mirror of
https://github.com/valitydev/riak_test.git
synced 2024-11-06 08:35:22 +00:00
f822e52fe8
ensemble_remove_node2 uses an intercept to prevent a riak_ensemble related transition that is necessary for nodes to completely exit and shutdown after removal. In fact, testing for this scenario is the entire point of this test, since it is testing logic that was added to solve basho/riak_core#572 and that logic prevents nodes from exiting until that transition occurs. However, even without this new logic, there is an unrelated riak_ensemble related bug that can trigger a race condition that also prevents nodes from shutting down. The good news is that other changes made as part of the solution to solve basho/riak_core#572 also fix this unrelated bug. Therefore this commit extends ensemble_remove_node2 to remove the intercept at the end of the test and verify that the removed nodes do actually end up exiting as expected. Thus, the test now tests for both the negative and positive scenarios and serves as a test against future regressions that stall node removal/shutdown.
107 lines
4.2 KiB
Erlang
107 lines
4.2 KiB
Erlang
%% -------------------------------------------------------------------
|
|
%%
|
|
%% Copyright (c) 2013-2014 Basho Technologies, Inc.
|
|
%%
|
|
%% This file is provided to you under the Apache License,
|
|
%% Version 2.0 (the "License"); you may not use this file
|
|
%% except in compliance with the License. You may obtain
|
|
%% a copy of the License at
|
|
%%
|
|
%% http://www.apache.org/licenses/LICENSE-2.0
|
|
%%
|
|
%% Unless required by applicable law or agreed to in writing,
|
|
%% software distributed under the License is distributed on an
|
|
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
%% KIND, either express or implied. See the License for the
|
|
%% specific language governing permissions and limitations
|
|
%% under the License.
|
|
%%
|
|
%% -------------------------------------------------------------------
|
|
|
|
-module(ensemble_remove_node2).
|
|
-export([confirm/0]).
|
|
-include_lib("eunit/include/eunit.hrl").
|
|
-compile({parse_transform, rt_intercept_pt}).
|
|
-define(M, riak_kv_ensemble_backend_orig).
|
|
|
|
confirm() ->
|
|
NumNodes = 3,
|
|
NVal = 3,
|
|
Config = ensemble_util:fast_config(NVal),
|
|
lager:info("Building cluster and waiting for ensemble to stablize"),
|
|
Nodes = ensemble_util:build_cluster(NumNodes, Config, NVal),
|
|
[Node, Node2, Node3] = Nodes,
|
|
ok = ensemble_util:wait_until_stable(Node, NVal),
|
|
lager:info("Store a value in the root ensemble"),
|
|
{ok, _} = riak_ensemble_client:kput_once(Node, root, testerooni,
|
|
testvalue, 1000),
|
|
lager:info("Read value from the root ensemble"),
|
|
{ok, _} = riak_ensemble_client:kget(Node, root, testerooni, 1000),
|
|
|
|
lager:info("Creating/activating 'strong' bucket type"),
|
|
rt:create_and_activate_bucket_type(Node, <<"strong">>,
|
|
[{consistent, true}, {n_val, NVal}]),
|
|
ensemble_util:wait_until_stable(Node, NVal),
|
|
Bucket = {<<"strong">>, <<"test">>},
|
|
Key = <<"testkey">>,
|
|
PBC = rt:pbc(Node),
|
|
ok = rt:pbc_write(PBC, Bucket, Key, testval),
|
|
Val1 = rt:pbc_read(PBC, Bucket, Key),
|
|
?assertEqual(element(1, Val1), riakc_obj),
|
|
|
|
%% Don't allow node deletions in riak_ensemble. This should prevent the
|
|
%% nodes from ever exiting
|
|
rt_intercept:add(Node, {riak_kv_ensemble_backend, [{{maybe_async_update, 2},
|
|
{[],
|
|
fun(Changes, State) ->
|
|
Changes2 = lists:filter(fun({del, _}) -> false;
|
|
(_) -> true
|
|
end, Changes),
|
|
?M:maybe_async_update_orig(Changes2, State)
|
|
end}}]}),
|
|
|
|
lager:info("Removing Nodes 2 and 3 from the cluster"),
|
|
rt:leave(Node2),
|
|
ok = ensemble_util:wait_until_stable(Node, NVal),
|
|
rt:leave(Node3),
|
|
ok = ensemble_util:wait_until_stable(Node, NVal),
|
|
Remaining = Nodes -- [Node2, Node3],
|
|
rt:wait_until_nodes_agree_about_ownership(Remaining),
|
|
|
|
%% TODO: How do we wait indefinitely for nodes to never exit here? A 30s
|
|
%% sleep?
|
|
timer:sleep(30000),
|
|
|
|
%% Nodes should still be in leaving state
|
|
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
|
|
Leaving = lists:usort(riak_core_ring:members(Ring, [leaving])),
|
|
?assertEqual(Leaving, [Node2, Node3]),
|
|
|
|
%% We should still be able to read from k/v ensembles, but the nodes should
|
|
%% never exit
|
|
lager:info("Reading From SC Bucket"),
|
|
Val2 = rt:pbc_read(PBC, Bucket, Key),
|
|
?assertEqual(element(1, Val2), riakc_obj),
|
|
|
|
ok = ensemble_util:wait_until_stable(Node, NVal),
|
|
lager:info("Read value from the root ensemble"),
|
|
{ok, _Obj} = riak_ensemble_client:kget(Node, root, testerooni, 1000),
|
|
Members3 = rpc:call(Node, riak_ensemble_manager, get_members, [root]),
|
|
?assertEqual(3, length(Members3)),
|
|
Cluster = rpc:call(Node, riak_ensemble_manager, cluster, []),
|
|
?assertEqual(3, length(Cluster)),
|
|
|
|
lager:info("Removing intercept and waiting until nodes 2/3 shutdown"),
|
|
rt_intercept:add(Node, {riak_kv_ensemble_backend, [{{maybe_async_update, 2},
|
|
{[],
|
|
fun(Changes, State) ->
|
|
?M:maybe_async_update_orig(Changes, State)
|
|
end}}]}),
|
|
|
|
ok = rt:wait_until_unpingable(Node2),
|
|
ok = rt:wait_until_unpingable(Node3),
|
|
rpc:call(Node, riak_core_console, member_status, [[]]),
|
|
rpc:call(Node, riak_core_console, ring_status, [[]]),
|
|
|
|
pass.
|