Merge pull request #106 from basho/jd-gh-core-155-tweaks

wait_for_owners_according_to

src/rt.erl

@@ -107,7 +107,9 @@
          wait_until_capability/3,
          wait_until_connected/1,
          wait_until_legacy_ringready/1,
+         wait_until_owners_according_to/2,
          wait_until_no_pending_changes/1,
+         wait_until_nodes_agree_about_ownership/1,
          wait_until_nodes_ready/1,
          wait_until_pingable/1,
          wait_until_ready/1,
@@ -227,6 +229,7 @@ async_start(Node) ->
 
 %% @doc Stop the specified Riak `Node'.
 stop(Node) ->
+    lager:info("Stopping riak on ~p", [Node]),
     rpc:call(Node, init, stop, []).
 
 %% @doc Stop the specified Riak `Node' and wait until it is not pingable
@@ -570,6 +573,17 @@ wait_until_capability(Node, Capability, Value) ->
                           Value == capability(Node, Capability)
                   end).
 
+wait_until_owners_according_to(Node, Nodes) ->
+    SortedNodes = lists:usort(Nodes),
+    F = fun(N) ->
+        owners_according_to(N) =:= SortedNodes
+    end,
+    ?assertEqual(ok, wait_until(Node, F)),
+    ok.
+
+wait_until_nodes_agree_about_ownership(Nodes) ->
+    Results = [ wait_until_owners_according_to(Node, Nodes) || Node <- Nodes ],
+    ?assert(lists:all(fun(X) -> ok =:= X end, Results)).
 %%%===================================================================
 %%% Ring Functions
 %%%===================================================================
@@ -588,6 +602,7 @@ get_ring(Node) ->
     Ring.
 
 assert_nodes_agree_about_ownership(Nodes) ->
+    ?assertEqual(ok, wait_until_ring_converged(Nodes)),
     ?assertEqual(ok, wait_until_all_members(Nodes)),
     [ ?assertEqual({Node, Nodes}, {Node, owners_according_to(Node)}) || Node <- Nodes].
 
@@ -656,7 +671,7 @@ build_cluster(NumNodes, Versions, InitialConfig) ->
     ?assertEqual(ok, wait_until_no_pending_changes(Nodes)),
 
     %% Ensure each node owns a portion of the ring
-    assert_nodes_agree_about_ownership(Nodes),
+    wait_until_nodes_agree_about_ownership(Nodes),
     lager:info("Cluster built: ~p", [Nodes]),
     Nodes.
 

tests/gh_riak_core_155.erl

@@ -59,15 +59,18 @@ confirm() ->
     pass.
 
 load_code(Module, Nodes) ->
-    {Module, Bin, File} = code:get_object_code(Module),
-    {_, []} = rpc:multicall(Nodes, code, load_binary, [Module, File, Bin]).
+    case code:get_object_code(Module) of
+        {Module, Bin, File} ->
+            {_, []} = rpc:multicall(Nodes, code, load_binary, [Module, File, Bin]);
+        error ->
+            error(lists:flatten(io_lib:format("unable to get_object_code(~s)", [Module])))
+    end.
 
 %% NOTE: Don't call lager in this function.  This function is compiled
 %% using the lager version specified by Riak Test's rebar.config but
 %% that may not match the version used by Riak where this function is
 %% called.
 setup_mocks() ->
-    application:start(lager),
     meck:new(riak_core_vnode_proxy_sup, [unstick, passthrough, no_link]),
     meck:expect(riak_core_vnode_proxy_sup, start_proxies,
                 fun(Mod=riak_kv_vnode) ->

tests/gh_riak_core_176.erl

@@ -57,7 +57,7 @@ confirm() ->
     rt:join(Node2, Node1),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes12)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes12)),
-    rt:assert_nodes_agree_about_ownership(Nodes12),
+    rt:wait_until_nodes_agree_about_ownership(Nodes12),
     
     %% Check 0.0.0.0 address works
     lager:info("Change ~p handoff_ip to \"0.0.0.0\"", [Node3]),
@@ -70,7 +70,7 @@ confirm() ->
     rt:join(Node3, Node1),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes123)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes123)),
-    rt:assert_nodes_agree_about_ownership(Nodes123),
+    rt:wait_until_nodes_agree_about_ownership(Nodes123),
 
     lager:info("Test gh_riak_core_176 passed"),
     pass.

tests/verify_build_cluster.erl

@@ -33,7 +33,8 @@ confirm() ->
 
     %% Ensure each node owns 100% of it's own ring
     lager:info("Ensure each nodes 100% of it's own ring"),
-    [?assertEqual([Node], rt:owners_according_to(Node)) || Node <- Nodes],
+
+    [rt:wait_until_owners_according_to(Node, [Node]) || Node <- Nodes],
 
     lager:info("Loading some data up in this cluster."),
     ?assertEqual([], rt:systest_write(Node1, 0, 1000, <<"verify_build_cluster">>, 2)),
@@ -96,7 +97,7 @@ wait_and_validate(RingNodes, UpNodes) ->
     ?assertEqual(ok, rt:wait_until_all_members(UpNodes)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(UpNodes)),
     lager:info("Ensure each node owns a portion of the ring"),
-    [?assertEqual(RingNodes, rt:owners_according_to(Node)) || Node <- UpNodes],
+    [rt:wait_until_owners_according_to(Node, RingNodes) || Node <- UpNodes],
     [rt:wait_for_service(Node, riak_kv) || Node <- UpNodes],
     lager:info("Verify that you got much data... (this is how we do it)"),
     ?assertEqual([], rt:systest_read(hd(UpNodes), 0, 1000, <<"verify_build_cluster">>, 2)),

tests/verify_leave.erl

@@ -24,7 +24,6 @@
 -import(rt, [build_cluster/1,
              leave/1,
              wait_until_unpingable/1,
-             owners_according_to/1,
              status_of_according_to/2,
              remove/2]).
 
@@ -42,7 +41,7 @@ confirm() ->
     lager:info("Verify ~p no longer owns partitions and all nodes believe "
                "it is invalid", [Node2]),
     Remaining1 = Nodes -- [Node2],
-    rt:assert_nodes_agree_about_ownership(Remaining1),
+    rt:wait_until_nodes_agree_about_ownership(Remaining1),
     [?assertEqual(invalid, status_of_according_to(Node2, Node)) || Node <- Remaining1],
 
     %% Have node1 remove node3
@@ -54,6 +53,6 @@ confirm() ->
     lager:info("Verify ~p no longer owns partitions, and all nodes believe "
                "it is invalid", [Node3]),
     Remaining2 = Remaining1 -- [Node3],
-    rt:assert_nodes_agree_about_ownership(Remaining2),
+    rt:wait_until_nodes_agree_about_ownership(Remaining2),
     [?assertEqual(invalid, status_of_according_to(Node3, Node)) || Node <- Remaining2],
     pass.