valitydev/riak_test
14
0
Fork 0
mirror of https://github.com/valitydev/riak_test.git synced 2024-11-06 08:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Initial release

Browse Source
This commit is contained in:
Joseph Blomstedt 2012-01-10 17:21:39 -07:00
commit a6aec227cd
15 changed files with 951 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Makefile Normal file
View File

@ -0,0 +1,16 @@
.PHONY: deps
all: deps compile
./rebar skip_deps=true escriptize
deps:
./rebar get-deps
compile: deps
./rebar compile
clean:
@./rebar clean
distclean: clean
@rm -rf riak_test deps

292
doc/overview.edoc Normal file
View File

@ -0,0 +1,292 @@
@doc
riak_test is a integration/system testing driver for Riak that runs
tests written in Erlang. You write the tests, and riak_test handles
bringing up a Riak cluster and running your tests against
it. riak_test can generate a cluster directly from a devrel build of a
Riak source tree, as well as integrate with basho_expect to generate
clusters from generated Riak packages.
The devrel mode operates against a generated devrel that has been
turned into a git repo, allowing riak_test to easily reset all nodes
by reverting the git repo back to the initial commit. This is the
preferred mode of operation as it is easy to setup and can easily test
a work-in-progress development tree during implementation and
code/automation review.
The basho_expect mode is designed to allow the same system tests to be
re-used for validating packages during the release cycle. In this
mode, riak_test launches basho_expect in a new server mode that
riak_test connects to and sends commands (eg. login to host, install
riak, start riak, etc). The server mode is based on Thrift and can
therefore be used from languages other than Erlang, say if you want to
control basho_expect from Ruby. The Thrift dependency could be removed
by finishing the pre-existing basho_expect Erlang port driver if
desired.
The following is an example of a riak_test test:
<pre class="erlang">
```
-import(rt, [deploy_nodes/1,
owners_according_to/1,
wait_until_nodes_ready/1,
wait_until_no_pending_changes/1]).
verify_build_cluster() ->
%% Deploy a set of new nodes
lager:info("Deploying 3 nodes"),
Nodes = deploy_nodes(3),
%% Ensure each node owns 100% of it's own ring
lager:info("Ensure each nodes 100% of it's own ring"),
[?assertEqual([Node], owners_according_to(Node)) || Node <- Nodes],
%% Join nodes
lager:info("Join nodes together"),
[Node1|OtherNodes] = Nodes,
[join(Node, Node1) || Node <- OtherNodes],
lager:info("Wait until all nodes are ready and there are no pending changes"),
?assertEqual(ok, wait_until_nodes_ready(Nodes)),
?assertEqual(ok, wait_until_no_pending_changes(Nodes)),
%% Ensure each node owns a portion of the ring
lager:info("Ensure each node owns a portion of the ring"),
[?assertEqual(Nodes, owners_according_to(Node)) || Node <- Nodes],
lager:info("verify_build_cluster: PASS"),
ok.
'''
</pre>
Running the test in devrel mode gives the following output:
<pre class="erlang">
```
16:10:27.787 [info] Application lager started on node 'riak_test@127.0.0.1'
16:10:27.792 [info] Deploying 3 nodes
16:10:27.792 [info] Running: /tmp/rt/dev/dev1/bin/riak stop
16:10:27.793 [info] Running: /tmp/rt/dev/dev2/bin/riak stop
16:10:27.793 [info] Running: /tmp/rt/dev/dev3/bin/riak stop
16:10:28.119 [info] Resetting nodes to fresh state
16:10:28.119 [debug] Running: git --git-dir="/tmp/rt/dev/.git" --work-tree="/tmp/rt/dev" reset HEAD --hard
16:10:34.313 [debug] Running: git --git-dir="/tmp/rt/dev/.git" --work-tree="/tmp/rt/dev" clean -fd
16:10:34.374 [info] Running: /tmp/rt/dev/dev1/bin/riak start
16:10:34.375 [info] Running: /tmp/rt/dev/dev2/bin/riak start
16:10:34.375 [info] Running: /tmp/rt/dev/dev3/bin/riak start
16:10:36.409 [debug] Supervisor inet_gethost_native_sup started undefined at pid <0.68.0>
16:10:36.411 [debug] Supervisor kernel_safe_sup started inet_gethost_native:start_link() at pid <0.67.0>
16:10:36.444 [info] Deployed nodes: ['dev1@127.0.0.1','dev2@127.0.0.1','dev3@127.0.0.1']
16:10:36.444 [info] Ensure each nodes 100% of it's own ring
16:10:36.445 [info] Join nodes together
16:10:36.465 [debug] [join] 'dev2@127.0.0.1' to ('dev1@127.0.0.1'): ok
16:10:36.491 [debug] [join] 'dev3@127.0.0.1' to ('dev1@127.0.0.1'): ok
16:10:36.491 [info] Wait until all nodes are ready and there are no pending changes
16:10:56.192 [info] Ensure each node owns a portion of the ring
16:10:56.193 [info] verify_build_cluster: PASS
'''
</pre>
Whereas running in basho_expect mode gives the following output:
```
16:20:13.066 [info] Application lager started on node 'riak_test@127.0.0.1'
16:20:13.086 [info] Connecting to basho_expect at {"127.0.0.1",9000}
16:20:13.090 [debug] Supervisor inet_gethost_native_sup started undefined at pid <0.53.0>
16:20:13.092 [debug] Supervisor kernel_safe_sup started inet_gethost_native:start_link() at pid <0.52.0>
16:20:13.100 [info] Logging into hosts: ["ubuntu10-64-1","ubuntu10-64-2","ubuntu10-64-3"]
<snip basho_expect output>
16:20:30.227 [info] Deploying 3 nodes
16:20:32.378 [info] Re-installing riak on nodes
<snip basho_expect output>
16:22:16.905 [info] Starting riak on nodes
<snip basho_expect output>
16:22:30.141 [info] Deployed nodes: ['riak@192.168.60.10','riak@192.168.60.11','riak@192.168.60.12']
16:22:30.141 [info] Ensure each nodes 100% of it's own ring
16:22:30.143 [info] Join nodes together
16:22:30.153 [debug] [join] 'riak@192.168.60.11' to ('riak@192.168.60.10'): ok
16:22:30.164 [debug] [join] 'riak@192.168.60.12' to ('riak@192.168.60.10'): ok
16:22:30.164 [info] Wait until all nodes are ready and there are no pending changes
16:22:49.433 [info] Ensure each node owns a portion of the ring
16:22:49.436 [info] verify_build_cluster: PASS
'''
As shown, the tests look similar to standard eunit tests and are
straightforward Erlang. The {@link rt} module provides the basic
commands to control Riak nodes, and abstracts everything away from
the underlying harness: devrel or basho_expect. The `rt' module
also provides a set of useful existing built-in operations. It is
expected that the `rt' module will be extended over time with more
reusable functions.
Writing additional checks and preconditions is the same as with an
eunit test. For example, `rt:owners_according_to(Node)' is just a
basic RPC call:
<pre class="erlang">
```
owners_according_to(Node) ->
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
Owners = [Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)],
lists:usort(Owners).
'''
</pre>
And the `rt:wait_until_nodes_ready', which leverages the built-in `wait_until'
primitive:
<pre>
```
wait_until_nodes_ready(Nodes) ->
[?assertEqual(ok, wait_until(Node, fun is_ready/1)) || Node <- Nodes],
ok.
is_ready(Node) ->
case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of
{ok, Ring} ->
lists:member(Node, riak_core_ring:ready_members(Ring));
_ ->
false
end.
'''
</pre>
Much like basho_bench, riak_test uses Erlang term based config files. The
references `rtdev.config' file above:
```
%% Deps should include the path to compiled Riak libs
{rt_deps, ["/Users/jtuple/basho/riak/deps"]}.
%% Maximum time in milliseconds for wait_until to wait
{rt_max_wait_time, 180000}.
%% Delay between each retry in wait_until
{rt_retry_delay, 500}.
%% Use the devrel harness
{rt_harness, rtdev}.
%% Path to generated devrel of Riak that is git-versioned
{rtdev_path, "/tmp/rt"}.
'''
And the `rtbe.config' file:
```
%% Deps should include Riak libs and basho_expect
{rt_deps, ["/Users/jtuple/basho/riak/deps",
"/Users/jtuple/basho/basho_expect"]}.
{rt_max_wait_time, 180000}.
{rt_retry_delay, 500}.
%% Use the rtbe harness (in basho_expect source tree)
{rt_harness, rtbe}.
%% Host/Port to connect to basho_expect server.
{rtbe_host, "127.0.0.1"}.
{rtbe_port, 9000}.
'''
And to conclude, a slightly longer test that uses most of the built-in
primitives including starting and stopping Riak nodes:
<pre class="erlang">
```
verify_claimant() ->
Nodes = build_cluster(3),
[Node1, Node2, _Node3] = Nodes,
%% Ensure all nodes believe node1 is the claimant
lager:info("Ensure all nodes believe ~p is the claimant", [Node1]),
[?assertEqual(Node1, claimant_according_to(Node)) || Node <- Nodes],
%% Stop node1
lager:info("Stop ~p", [Node1]),
stop(Node1),
?assertEqual(ok, wait_until_unpingable(Node1)),
%% Ensure all nodes still believe node1 is the claimant
lager:info("Ensure all nodes still believe ~p is the claimant", [Node1]),
Remaining = Nodes -- [Node1],
[?assertEqual(Node1, claimant_according_to(Node)) || Node <- Remaining],
%% Mark node1 as down and wait for ring convergence
lager:info("Mark ~p as down", [Node1]),
down(Node2, Node1),
?assertEqual(ok, wait_until_ring_converged(Remaining)),
[?assertEqual(down, status_of_according_to(Node1, Node)) || Node <- Remaining],
%% Ensure all nodes now believe node2 to be the claimant
lager:info("Ensure all nodes now believe ~p is the claimant", [Node2]),
[?assertEqual(Node2, claimant_according_to(Node)) || Node <- Remaining],
%% Restart node1 and wait for ring convergence
lager:info("Restart ~p and wait for ring convergence", [Node1]),
start(Node1),
?assertEqual(ok, wait_until_nodes_ready([Node1])),
?assertEqual(ok, wait_until_ring_converged(Nodes)),
%% Ensure node has rejoined and is no longer down
lager:info("Ensure ~p has rejoined and is no longer down", [Node1]),
[?assertEqual(valid, status_of_according_to(Node1, Node)) || Node <- Nodes],
%% Ensure all nodes still believe node2 is the claimant
lager:info("Ensure all nodes still believe ~p is the claimant", [Node2]),
[?assertEqual(Node2, claimant_according_to(Node)) || Node <- Nodes],
ok.
'''
</pre>
<h2>Installation and Usage</h2>
Checkout riak_test from Github and build: `http://github.com/basho/riak_test'
riak_test comes with two pre-written config files: `rtdev.config' for
devrel mode and `rtbe.config' for basho_expect mode. The devrel
operating mode comes built into riak_test, and can be used right away.
To use the basho_expect mode, you must checkout a version of basho_expect that
includes the basho_expect_thrift server mode, and make sure to run `make' from
within the `basho_expect/basho_expect_thrift' directory.
For both modes, ensure the `rt_deps' setting in the config file is accurate,
and adjust other settings as appropriate. The comments should be self explanatory.
To run riak_test in devrel mode, you must first generate a git-versioned devrel
directory and ensure the `rtdev_path' setting in `rtdev.config' is accurate.
To generate the versioned devrel, modify the following as necessary:
```
riak$ make devrel
riak$ mkdir /tmp/rt
riak$ cp -r dev /tmp/rt/dev
riak$ cd /tmp/rt/dev
/tmp/rt/dev$ git init
/tmp/rt/dev$ git add .
/tmp/rt/dev$ git commit -m "initial"
'''
You can then launch tests from within the riak_test directory:
```
./riak_test rtdev.config verify_build_cluster
'''
riak_test currently assumes tests are written such that there is one
test per module and that the module contains a same named zero arity
function that implements the test. For example, the above calls
`verify_build_cluster:verify_build_cluster()'.
To run run in basho_expect mode, you must first launch the basho_expect
server and then run riak_test:
```
basho_expect$ python run_be_thrift.py &
riak_test$ ./riak_test rtbe.config verify_build_cluster ubuntu10-64-1 ubuntu10-64-2 ubuntu10-64-3
'''
To make this easier, there is an included script in basho_expect: `Run-riak-test' that operates
more like traditional basho_expect tests. Just make sure to modify it to point to the right
riak_test directory. The above simplifies to the following:
```
basho_expect$ ./Run-riak-test verify_build_cluster ubuntu10-64-1 ubuntu10-64-2 ubuntu10-64-3
'''
TODO: integrate with general basho_expect systest harness and ensure things work with
Guttersnipe.

BIN
rebar vendored Executable file
View File

Binary file not shown.

14
rebar.config Normal file
View File

@ -0,0 +1,14 @@
{require_otp_vsn, "R13B04|R14"}.
{cover_enabled, true}.
{edoc_opts, [{preprocess, true}]}.
%%{edoc_opts, [{doclet, edown_doclet}, {pretty_printer, erl_pp}]}.
%%{edoc_opts, [{doclet, my_layout}, {pretty_printer, erl_pp}]}.
%%{edoc_opts, [{layout, my_layout}, {file_suffix, ".xml"}, {pretty_printer, erl_pp}]}.
{erl_opts, [{src_dirs, [src, tests]},
warnings_as_errors, {parse_transform, lager_transform}]}.
{deps, [
{lager, ".*", {git, "git://github.com/basho/lager", {branch, "master"}}}
]}.
{escript_incl_apps, [lager]}.

7
rtbe.config Normal file
View File

@ -0,0 +1,7 @@
{rt_deps, ["/Users/jtuple/basho/CLEAN2/riak/deps",
"/Users/jtuple/basho/basho_expect"]}.
{rt_max_wait_time, 180000}.
{rt_retry_delay, 500}.
{rt_harness, rtbe}.
{rtbe_host, "127.0.0.1"}.
{rtbe_port, 9000}.

7
rtdev.config Normal file
View File

@ -0,0 +1,7 @@
{rt_deps, ["/Users/jtuple/basho/CLEAN2/riak/deps"]}.
{rt_max_wait_time, 180000}.
{rt_retry_delay, 500}.
{rt_harness, rtdev}.
% {rtdev_path, "/Users/jtuple/basho/CLEAN2/riak"}.
{rtdev_path, "/tmp/rt"}.

11
src/riak_test.app.src Normal file
View File

@ -0,0 +1,11 @@
{application, riak_test,
[
{description, ""},
{vsn, "0.1.0"},
{registered, []},
{applications, [
kernel,
stdlib
]},
{env, []}
]}.

33
src/riak_test.erl Normal file
View File

@ -0,0 +1,33 @@
%% @private
-module(riak_test).
-export([main/1]).
add_deps(Path) ->
{ok, Deps} = file:list_dir(Path),
[code:add_path(lists:append([Path, "/", Dep, "/ebin"])) || Dep <- Deps],
ok.
main(Args) ->
[Config, Test | HarnessArgs]=Args,
rt:load_config(Config),
[add_deps(Dep) || Dep <- rt:config(rt_deps)],
ENode = rt:config(rt_nodename, 'riak_test@127.0.0.1'),
Cookie = rt:config(rt_cookie, riak),
[] = os:cmd("epmd -daemon"),
net_kernel:start([ENode]),
erlang:set_cookie(node(), Cookie),
application:start(lager),
lager:set_loglevel(lager_console_backend, debug),
%% rt:set_config(rtdev_path, Path),
%% rt:set_config(rt_max_wait_time, 180000),
%% rt:set_config(rt_retry_delay, 500),
%% rt:set_config(rt_harness, rtbe),
rt:setup_harness(Test, HarnessArgs),
TestA = list_to_atom(Test),
%% st:TestFn(),
TestA:TestA(),
rt:cleanup_harness(),
ok.

289
src/rt.erl Normal file
View File

@ -0,0 +1,289 @@
%% @doc
%% Implements the base `riak_test' API, providing the ability to control
%% nodes in a Riak cluster as well as perform commonly reused operations.
%% Please extend this module with new functions that prove useful between
%% multiple independent tests.
-module(rt).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-export([deploy_nodes/1,
start/1,
stop/1,
join/2,
leave/1,
wait_until_pingable/1,
wait_until_unpingable/1,
wait_until_ready/1,
wait_until_no_pending_changes/1,
wait_until_nodes_ready/1,
remove/2,
down/2,
check_singleton_node/1,
owners_according_to/1,
members_according_to/1,
status_of_according_to/2,
claimant_according_to/1,
wait_until_all_members/1,
wait_until_all_members/2,
wait_until_ring_converged/1]).
-export([setup_harness/2,
cleanup_harness/0,
load_config/1,
set_config/2,
config/1,
config/2
]).
-define(HARNESS, (rt:config(rt_harness))).
%% @doc Deploy a set of freshly installed Riak nodes, returning a list of the
%% nodes deployed.
-spec deploy_nodes(NumNodes :: integer()) -> [node()].
deploy_nodes(NumNodes) ->
?HARNESS:deploy_nodes(NumNodes).
%% @doc Start the specified Riak node
start(Node) ->
?HARNESS:start(Node).
%% @doc Stop the specified Riak node
stop(Node) ->
?HARNESS:stop(Node).
%% @doc Have `Node' send a join request to `PNode'
join(Node, PNode) ->
R = rpc:call(Node, riak_core, join, [PNode]),
lager:debug("[join] ~p to (~p): ~p", [Node, PNode, R]),
%% wait_until_ready(Node),
ok.
%% @doc Have the specified node leave the cluster
leave(Node) ->
R = rpc:call(Node, riak_core, leave, []),
lager:debug("[leave] ~p: ~p", [Node, R]),
ok.
%% @doc Have `Node' remove `OtherNode' from the cluster
remove(Node, OtherNode) ->
rpc:call(Node, riak_kv_console, remove, [[atom_to_list(OtherNode)]]).
%% @doc Have `Node' mark `OtherNode' as down
down(Node, OtherNode) ->
rpc:call(Node, riak_kv_console, down, [[atom_to_list(OtherNode)]]).
%% @doc Ensure that the specified node is a singleton node/cluster -- a node
%% that owns 100% of the ring.
check_singleton_node(Node) ->
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
Owners = lists:usort([Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)]),
?assertEqual([Node], Owners),
ok.
%% @doc Wait until the specified node is considered ready by `riak_core'.
%% As of Riak 1.0, a node is ready if it is in the `valid' or `leaving'
%% states. A ready node is guaranteed to have current preflist/ownership
%% information.
wait_until_ready(Node) ->
?assertEqual(ok, wait_until(Node, fun is_ready/1)),
ok.
%% @doc Given a list of nodes, wait until all nodes believe there are no
%% on-going or pending ownership transfers.
-spec wait_until_no_pending_changes([node()]) -> ok | fail.
wait_until_no_pending_changes(Nodes) ->
F = fun(Node) ->
[rpc:call(NN, riak_core_vnode_manager, force_handoffs, [])
|| NN <- Nodes],
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
riak_core_ring:pending_changes(Ring) =:= []
end,
[?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes],
ok.
%% @private
are_no_pending(Node) ->
rpc:call(Node, riak_core_vnode_manager, force_handoffs, []),
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
riak_core_ring:pending_changes(Ring) =:= [].
%% @doc Return a list of nodes that own partitions according to the ring
%% retrieved from the specified node.
owners_according_to(Node) ->
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
Owners = [Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)],
lists:usort(Owners).
%% @doc Return a list of cluster members according to the ring retrieved from
%% the specified node.
members_according_to(Node) ->
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
Members = riak_core_ring:all_members(Ring),
Members.
%% @doc Return the cluster status of `Member' according to the ring
%% retrieved from `Node'.
status_of_according_to(Member, Node) ->
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
Status = riak_core_ring:member_status(Ring, Member),
Status.
%% @doc Return a list of nodes that own partitions according to the ring
%% retrieved from the specified node.
claimant_according_to(Node) ->
{ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
Claimant = riak_core_ring:claimant(Ring),
Claimant.
%% @doc Given a list of nodes, wait until all nodes are considered ready.
%% See {@link wait_until_ready/1} for definition of ready.
wait_until_nodes_ready(Nodes) ->
[?assertEqual(ok, wait_until(Node, fun is_ready/1)) || Node <- Nodes],
ok.
%% @private
is_ready(Node) ->
case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of
{ok, Ring} ->
lists:member(Node, riak_core_ring:ready_members(Ring));
_ ->
false
end.
%% @doc Wait until all nodes in the list `Nodes' believe each other to be
%% members of the cluster.
wait_until_all_members(Nodes) ->
wait_until_all_members(Nodes, Nodes).
%% @doc Wait until all nodes in the list `Nodes' believes all nodes in the
%% list `Members' are members of the cluster.
wait_until_all_members(Nodes, Members) ->
S1 = ordsets:from_list(Members),
F = fun(Node) ->
S2 = ordsets:from_list(members_according_to(Node)),
ordsets:is_subset(S1, S2)
end,
[?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes],
ok.
%% @doc Given a list of nodes, wait until all nodes believe the ring has
%% converged (ie. `riak_core_ring:is_ready' returns `true').
wait_until_ring_converged(Nodes) ->
[?assertEqual(ok, wait_until(Node, fun is_ring_ready/1)) || Node <- Nodes],
ok.
%% @private
is_ring_ready(Node) ->
case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of
{ok, Ring} ->
riak_core_ring:ring_ready(Ring);
_ ->
false
end.
%% @doc Wait until the specified node is pingable
wait_until_pingable(Node) ->
F = fun(N) ->
net_adm:ping(N) =:= pong
end,
?assertEqual(ok, wait_until(Node, F)),
ok.
%% @doc Wait until the specified node is no longer pingable
wait_until_unpingable(Node) ->
F = fun(N) ->
net_adm:ping(N) =:= pang
end,
?assertEqual(ok, wait_until(Node, F)),
ok.
%% @doc Utility function used to construct test predicates. Retries the
%% function `Fun' until it returns `true', or until the maximum
%% number of retries is reached. The retry limit is based on the
%% provided `rt_max_wait_time' and `rt_retry_delay' parameters in
%% specified `riak_test' config file.
wait_until(Node, Fun) ->
MaxTime = rt:config(rt_max_wait_time),
Delay = rt:config(rt_retry_delay),
Retry = MaxTime div Delay,
wait_until(Node, Fun, Retry, Delay).
%% @deprecated Use {@link wait_until/2} instead.
wait_until(Node, Fun, Retry) ->
wait_until(Node, Fun, Retry, 500).
%% @deprecated Use {@link wait_until/2} instead.
wait_until(Node, Fun, Retry, Delay) ->
Pass = Fun(Node),
case {Retry, Pass} of
{_, true} ->
ok;
{0, _} ->
fail;
_ ->
timer:sleep(Delay),
wait_until(Node, Fun, Retry-1)
end.
%% @private
setup_harness(Test, Args) ->
?HARNESS:setup_harness(Test, Args).
%% @private
cleanup_harness() ->
?HARNESS:cleanup_harness().
%% @private
load_config(File) ->
case file:consult(File) of
{ok, Terms} ->
[set_config(Key, Value) || {Key, Value} <- Terms],
ok;
{error, Reason} ->
erlang:error("Failed to parse config file", [File, Reason])
end.
%% @private
set_config(Key, Value) ->
ok = application:set_env(riak_test, Key, Value).
%% @private
config(Key) ->
case application:get_env(riak_test, Key) of
{ok, Value} ->
Value;
undefined ->
erlang:error("Missing configuration key", [Key])
end.
%% @private
config(Key, Default) ->
case application:get_env(riak_test, Key) of
{ok, Value} ->
Value;
_ ->
Default
end.
%% @doc
%% Safely construct a `NumNode' size cluster and return a list of the
%% deployed nodes.
build_cluster(NumNodes) ->
%% Deploy a set of new nodes
Nodes = deploy_nodes(NumNodes),
%% Ensure each node owns 100% of it's own ring
[?assertEqual([Node], owners_according_to(Node)) || Node <- Nodes],
%% Join nodes
[Node1|OtherNodes] = Nodes,
[join(Node, Node1) || Node <- OtherNodes],
?assertEqual(ok, wait_until_nodes_ready(Nodes)),
?assertEqual(ok, wait_until_no_pending_changes(Nodes)),
%% Ensure each node owns a portion of the ring
[?assertEqual(Nodes, owners_according_to(Node)) || Node <- Nodes],
lager:info("Cluster built: ~p", [Nodes]),
Nodes.

96
src/rtdev.erl Normal file
View File

@ -0,0 +1,96 @@
%% @private
-module(rtdev).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])).
-define(DEV(N), list_to_atom(?DEVS(N))).
-define(PATH, (rt:config(rtdev_path))).
riakcmd(Path, N, Cmd) ->
io_lib:format("~s/dev/dev~b/bin/riak ~s", [Path, N, Cmd]).
gitcmd(Path, Cmd) ->
io_lib:format("git --git-dir=\"~s/dev/.git\" --work-tree=\"~s/dev\" ~s",
[Path, Path, Cmd]).
run_git(Path, Cmd) ->
lager:debug("Running: ~s", [gitcmd(Path, Cmd)]),
os:cmd(gitcmd(Path, Cmd)).
run_riak(N, Path, Cmd) ->
%% io:format("~p~n", [riakcmd(Path, N, Cmd)]),
%%?debugFmt("RR: ~p~n", [[N,Path,Cmd]]),
%%?debugFmt("~p~n", [os:cmd(riakcmd(Path, N, Cmd))]).
lager:info("Running: ~s", [riakcmd(Path, N, Cmd)]),
os:cmd(riakcmd(Path, N, Cmd)).
setup_harness(_Test, _Args) ->
ok.
cleanup_harness() ->
ok.
deploy_nodes(NumNodes) ->
Path = ?PATH,
lager:info("Riak path: ~p", [Path]),
NodesN = lists:seq(1, NumNodes),
Nodes = [?DEV(N) || N <- NodesN],
NodeMap = orddict:from_list(lists:zip(Nodes, NodesN)),
rt:set_config(rt_nodes, NodeMap),
%% Stop nodes if already running
%% [run_riak(N, Path, "stop") || N <- Nodes],
%%rpc:pmap({?MODULE, run_riak}, [Path, "stop"], Nodes),
pmap(fun(N) -> run_riak(N, Path, "stop") end, NodesN),
%% ?debugFmt("Shutdown~n", []),
%% Reset nodes to base state
lager:info("Resetting nodes to fresh state"),
%% run_git(Path, "status"),
run_git(Path, "reset HEAD --hard"),
run_git(Path, "clean -fd"),
%% run_git(Path, "status"),
%% ?debugFmt("Reset~n", []),
%% Start nodes
%%[run_riak(N, Path, "start") || N <- Nodes],
%%rpc:pmap({?MODULE, run_riak}, [Path, "start"], Nodes),
pmap(fun(N) -> run_riak(N, Path, "start") end, NodesN),
%% Ensure nodes started
[ok = rt:wait_until_pingable(N) || N <- Nodes],
%% %% Enable debug logging
%% [rpc:call(N, lager, set_loglevel, [lager_console_backend, debug]) || N <- Nodes],
%% Ensure nodes are singleton clusters
[ok = rt:check_singleton_node(N) || N <- Nodes],
lager:info("Deployed nodes: ~p", [Nodes]),
Nodes.
stop(Node) ->
run_riak(node_id(Node), ?PATH, "stop"),
ok.
start(Node) ->
run_riak(node_id(Node), ?PATH, "start"),
ok.
node_id(Node) ->
NodeMap = rt:config(rt_nodes),
orddict:fetch(Node, NodeMap).
pmap(F, L) ->
Parent = self(),
lists:foldl(
fun(X, N) ->
spawn(fun() ->
Parent ! {pmap, N, F(X)}
end),
N+1
end, 0, L),
L2 = [receive {pmap, N, R} -> {N,R} end || _ <- L],
{_, L3} = lists:unzip(lists:keysort(1, L2)),
L3.

9
tests/rt_basic_test.erl Normal file
View File

@ -0,0 +1,9 @@
-module(rt_basic_test).
-export([rt_basic_test/0]).
rt_basic_test() ->
lager:info("Deploy some nodes"),
Nodes = rt:deploy_nodes(2),
lager:info("Stop the nodes"),
[rt:stop(Node) || Node <- Nodes],
ok.

33
tests/verify_build_cluster.erl Normal file
View File

@ -0,0 +1,33 @@
-module(verify_build_cluster).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-import(rt, [deploy_nodes/1,
owners_according_to/1,
join/2,
wait_until_nodes_ready/1,
wait_until_no_pending_changes/1]).
verify_build_cluster() ->
%% Deploy a set of new nodes
lager:info("Deploying 3 nodes"),
Nodes = deploy_nodes(3),
%% Ensure each node owns 100% of it's own ring
lager:info("Ensure each nodes 100% of it's own ring"),
[?assertEqual([Node], owners_according_to(Node)) || Node <- Nodes],
%% Join nodes
lager:info("Join nodes together"),
[Node1|OtherNodes] = Nodes,
[join(Node, Node1) || Node <- OtherNodes],
lager:info("Wait until all nodes are ready and there are no pending changes"),
?assertEqual(ok, wait_until_nodes_ready(Nodes)),
?assertEqual(ok, wait_until_no_pending_changes(Nodes)),
%% Ensure each node owns a portion of the ring
lager:info("Ensure each node owns a portion of the ring"),
[?assertEqual(Nodes, owners_according_to(Node)) || Node <- Nodes],
lager:info("verify_build_cluster: PASS"),
ok.

56
tests/verify_claimant.erl Normal file
View File

@ -0,0 +1,56 @@
-module(verify_claimant).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-import(rt, [build_cluster/1,
start/1,
stop/1,
down/2,
claimant_according_to/1,
wait_until_unpingable/1,
wait_until_ring_converged/1,
status_of_according_to/2,
wait_until_nodes_ready/1]).
verify_claimant() ->
Nodes = build_cluster(3),
[Node1, Node2, _Node3] = Nodes,
%% Ensure all nodes believe node1 is the claimant
lager:info("Ensure all nodes believe ~p is the claimant", [Node1]),
[?assertEqual(Node1, claimant_according_to(Node)) || Node <- Nodes],
%% Stop node1
lager:info("Stop ~p", [Node1]),
stop(Node1),
?assertEqual(ok, wait_until_unpingable(Node1)),
%% Ensure all nodes still believe node1 is the claimant
lager:info("Ensure all nodes still believe ~p is the claimant", [Node1]),
Remaining = Nodes -- [Node1],
[?assertEqual(Node1, claimant_according_to(Node)) || Node <- Remaining],
%% Mark node1 as down and wait for ring convergence
lager:info("Mark ~p as down", [Node1]),
down(Node2, Node1),
?assertEqual(ok, wait_until_ring_converged(Remaining)),
[?assertEqual(down, status_of_according_to(Node1, Node)) || Node <- Remaining],
%% Ensure all nodes now believe node2 to be the claimant
lager:info("Ensure all nodes now believe ~p is the claimant", [Node2]),
[?assertEqual(Node2, claimant_according_to(Node)) || Node <- Remaining],
%% Restart node1 and wait for ring convergence
lager:info("Restart ~p and wait for ring convergence", [Node1]),
start(Node1),
?assertEqual(ok, wait_until_nodes_ready([Node1])),
?assertEqual(ok, wait_until_ring_converged(Nodes)),
%% Ensure node has rejoined and is no longer down
lager:info("Ensure ~p has rejoined and is no longer down", [Node1]),
[?assertEqual(valid, status_of_according_to(Node1, Node)) || Node <- Nodes],
%% Ensure all nodes still believe node2 is the claimant
lager:info("Ensure all nodes still believe ~p is the claimant", [Node2]),
[?assertEqual(Node2, claimant_according_to(Node)) || Node <- Nodes],
ok.

48
tests/verify_down.erl Normal file
View File

@ -0,0 +1,48 @@
-module(verify_down).
-export([verify_down/0]).
-include_lib("eunit/include/eunit.hrl").
verify_down() ->
Nodes = rt:deploy_nodes(3),
[Node1, Node2, Node3] = Nodes,
%% Join node2 to node1 and wait for cluster convergence
lager:info("Join ~p to ~p", [Node2, Node1]),
rt:join(Node2, Node1),
?assertEqual(ok, rt:wait_until_nodes_ready([Node1, Node2])),
?assertEqual(ok, rt:wait_until_no_pending_changes([Node1, Node2])),
%% Shutdown node2
lager:info("Stopping ~p", [Node2]),
rt:stop(Node2),
?assertEqual(ok, rt:wait_until_unpingable(Node2)),
Remaining = Nodes -- [Node2],
%% Join node3 to node1
lager:info("Join ~p to ~p", [Node3, Node1]),
rt:join(Node3, Node1),
?assertEqual(ok, rt:wait_until_all_members(Remaining, [Node3])),
%% Ensure node3 remains in the joining state
lager:info("Ensure ~p remains in the joining state", [Node3]),
[?assertEqual(joining, rt:status_of_according_to(Node3, Node)) || Node <- Remaining],
%% Mark node2 as down and wait for ring convergence
lager:info("Mark ~p as down", [Node2]),
rt:down(Node1, Node2),
?assertEqual(ok, rt:wait_until_ring_converged(Remaining)),
[?assertEqual(down, rt:status_of_according_to(Node2, Node)) || Node <- Remaining],
%% Ensure node3 is now valid
[?assertEqual(valid, rt:status_of_according_to(Node3, Node)) || Node <- Remaining],
%% Restart node2 and wait for ring convergence
lager:info("Restart ~p and wait for ring convergence", [Node2]),
rt:start(Node2),
?assertEqual(ok, rt:wait_until_nodes_ready([Node2])),
?assertEqual(ok, rt:wait_until_ring_converged(Nodes)),
%% Verify that all three nodes are ready
lager:info("Ensure all nodes are ready"),
?assertEqual(ok, rt:wait_until_nodes_ready(Nodes)),
ok.

40
tests/verify_leave.erl Normal file
View File

@ -0,0 +1,40 @@
-module(verify_leave).
-export([verify_leave/0]).
-include_lib("eunit/include/eunit.hrl").
-import(rt, [build_cluster/1,
leave/1,
wait_until_unpingable/1,
owners_according_to/1,
status_of_according_to/2,
remove/2]).
verify_leave() ->
%% Bring up a 3-node cluster for the test
Nodes = build_cluster(3),
[Node1, Node2, Node3] = Nodes,
%% Have node2 leave
lager:info("Have ~p leave", [Node2]),
leave(Node2),
?assertEqual(ok, wait_until_unpingable(Node2)),
%% Verify node2 no longer owns partitions, all node believe it invalid
lager:info("Verify ~p no longer owns partitions and all nodes believe "
"it is invalid", [Node2]),
Remaining1 = Nodes -- [Node2],
[?assertEqual(Remaining1, owners_according_to(Node)) || Node <- Remaining1],
[?assertEqual(invalid, status_of_according_to(Node2, Node)) || Node <- Remaining1],
%% Have node1 remove node3
lager:info("Have ~p remove ~p", [Node1, Node3]),
remove(Node1, Node3),
?assertEqual(ok, wait_until_unpingable(Node3)),
%% Verify node3 no longer owns partitions, all node believe it invalid
lager:info("Verify ~p no longer owns partitions, and all nodes believe "
"it is invalid", [Node3]),
Remaining2 = Remaining1 -- [Node3],
[?assertEqual(Remaining2, owners_according_to(Node)) || Node <- Remaining2],
[?assertEqual(invalid, status_of_according_to(Node3, Node)) || Node <- Remaining2],
ok.