Merge pull request #101 from basho/jd-owners_according_to

added rt:assert_nodes_agree_about_ownership/1, and there was much rejoicing

src/rt.erl

@@ -28,6 +28,7 @@
 
 -export([
          admin/2,
+         assert_nodes_agree_about_ownership/1,
          async_start/1,
          attach/2,
          build_cluster/1,
@@ -586,6 +587,10 @@ get_ring(Node) ->
     {ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
     Ring.
 
+assert_nodes_agree_about_ownership(Nodes) ->
+    ?assertEqual(ok, wait_until_all_members(Nodes)),
+    [ ?assertEqual({Node, Nodes}, {Node, owners_according_to(Node)}) || Node <- Nodes].
+
 %% @doc Return a list of nodes that own partitions according to the ring
 %%      retrieved from the specified node.
 owners_according_to(Node) ->
@@ -651,7 +656,7 @@ build_cluster(NumNodes, Versions, InitialConfig) ->
     ?assertEqual(ok, wait_until_no_pending_changes(Nodes)),
 
     %% Ensure each node owns a portion of the ring
-    [?assertEqual(Nodes, owners_according_to(Node)) || Node <- Nodes],
+    assert_nodes_agree_about_ownership(Nodes),
     lager:info("Cluster built: ~p", [Nodes]),
     Nodes.
 

tests/gh_riak_core_176.erl

@@ -47,6 +47,7 @@ confirm() ->
                [Node2, NodeIP, AlternateIP]),
     NewConfig = [{riak_core, [{handoff_ip, AlternateIP}]}],
     rt:update_app_config(Node2, NewConfig),
+    rt:wait_for_service(Node2, riak_kv),
 
     lager:info("Write data to the cluster"),
     rt:systest_write(Node1, 100),
@@ -56,8 +57,8 @@ confirm() ->
     rt:join(Node2, Node1),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes12)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes12)),
-    [?assertEqual(Nodes12, rt:owners_according_to(Node)) || Node <- Nodes12],
-
+    rt:assert_nodes_agree_about_ownership(Nodes12),
+    
     %% Check 0.0.0.0 address works
     lager:info("Change ~p handoff_ip to \"0.0.0.0\"", [Node3]),
     rt:update_app_config(Node3,
@@ -65,10 +66,11 @@ confirm() ->
 
     lager:info("Join ~p to the cluster and wait for handoff to finish",
                [Node3]),
+    rt:wait_for_service(Node3, riak_kv),
     rt:join(Node3, Node1),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes123)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes123)),
-    [?assertEqual(Nodes123, rt:owners_according_to(Node)) || Node <- Nodes123],
+    rt:assert_nodes_agree_about_ownership(Nodes123),
 
     lager:info("Test gh_riak_core_176 passed"),
     pass.

tests/verify_claimant.erl

@@ -63,6 +63,7 @@ confirm() ->
     lager:info("Restart ~p and wait for ring convergence", [Node1]),
     start(Node1),
     ?assertEqual(ok, wait_until_nodes_ready([Node1])),
+    ?assertEqual(ok, rt:wait_until_all_members(Nodes)),
     ?assertEqual(ok, wait_until_ring_converged(Nodes)),
 
     %% Ensure node has rejoined and is no longer down

tests/verify_leave.erl

@@ -42,7 +42,7 @@ confirm() ->
     lager:info("Verify ~p no longer owns partitions and all nodes believe "
                "it is invalid", [Node2]),
     Remaining1 = Nodes -- [Node2],
-    [?assertEqual(Remaining1, owners_according_to(Node)) || Node <- Remaining1],
+    rt:assert_nodes_agree_about_ownership(Remaining1),
     [?assertEqual(invalid, status_of_according_to(Node2, Node)) || Node <- Remaining1],
 
     %% Have node1 remove node3
@@ -54,6 +54,6 @@ confirm() ->
     lager:info("Verify ~p no longer owns partitions, and all nodes believe "
                "it is invalid", [Node3]),
     Remaining2 = Remaining1 -- [Node3],
-    [?assertEqual(Remaining2, owners_according_to(Node)) || Node <- Remaining2],
+    rt:assert_nodes_agree_about_ownership(Remaining2),
     [?assertEqual(invalid, status_of_according_to(Node3, Node)) || Node <- Remaining2],
     pass.

tests/verify_staged_clustering.erl

@@ -46,7 +46,7 @@ confirm() ->
     lager:info("Ensure that ~p now own all partitions", [Nodes123]),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes123)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes123)),
-    [?assertEqual(Nodes123, rt:owners_according_to(Node)) || Node <- Nodes123],
+    rt:assert_nodes_agree_about_ownership(Nodes123),
 
     lager:info("Join ~p to the cluster", [Node4]),
     stage_join(Node4, Node1),
@@ -63,8 +63,8 @@ confirm() ->
     lager:info("Ensure that ~p now own all partitions", [Nodes134]),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes134)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes134)),
-    [?assertEqual(Nodes134, rt:owners_according_to(Node)) || Node <- Nodes134],
-
+    rt:assert_nodes_agree_about_ownership(Nodes134),
+    
     lager:info("Verify that ~p shutdown after being replaced", [Node2]),
     ?assertEqual(ok, rt:wait_until_unpingable(Node2)),
 
@@ -84,7 +84,7 @@ confirm() ->
     lager:info("Ensure that ~p now own all partitions", [Nodes124]),
     ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes124)),
     ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes124)),
-    [?assertEqual(Nodes124, rt:owners_according_to(Node)) || Node <- Nodes124],
+    rt:assert_nodes_agree_about_ownership(Nodes124),
 
     lager:info("Stage leave of ~p", [Node2]),
     stage_leave(Node1, Node2),