riak_test/doc/overview.edoc

@doc
riak_test is a integration/system testing driver for Riak that runs
tests written in Erlang. You write the tests, and riak_test handles
bringing up a Riak cluster and running your tests against
it. riak_test can generate a cluster directly from a devrel build of a
Riak source tree.

The devrel mode operates against a generated devrel that has been
turned into a git repo, allowing riak_test to easily reset all nodes
by reverting the git repo back to the initial commit. This is the
preferred mode of operation as it is easy to setup and can easily test
a work-in-progress development tree during implementation and
code/automation review.

The following is an example of a riak_test test:

<pre class="erlang">
```
-module(verify_build_cluster).
-compile([confirm/0]).
-include_lib("eunit/include/eunit.hrl").

-import(rt, [deploy_nodes/1,
             owners_according_to/1,
             join/2,
             wait_until_nodes_ready/1,
             wait_until_no_pending_changes/1]).

confirm() ->
    %% Deploy a set of new nodes
    lager:info("Deploying 3 nodes"),
    Nodes = rt:deploy_nodes(3),

    %% Ensure each node owns 100% of it's own ring
    lager:info("Ensure each nodes 100% of it's own ring"),
    [?assertEqual([Node], owners_according_to(Node)) || Node <- Nodes],

    %% Join nodes
    lager:info("Join nodes together"),
    [Node1|OtherNodes] = Nodes,
    [join(Node, Node1) || Node <- OtherNodes],

    lager:info("Wait until all nodes are ready and there are no pending changes"),
    ?assertEqual(ok, wait_until_nodes_ready(Nodes)),
    ?assertEqual(ok, wait_until_no_pending_changes(Nodes)),

    %% Ensure each node owns a portion of the ring
    lager:info("Ensure each node owns a portion of the ring"),
    [?assertEqual(Nodes, owners_according_to(Node)) || Node <- Nodes],
    lager:info("verify_build_cluster: PASS"),
    pass.
'''
</pre>

Running the test in devrel mode gives the following output:

<pre class="erlang">
```
08:56:50.474 [info] Resetting nodes to fresh state
08:56:50.474 [debug] Running: git --git-dir="~/rt/riak/.git" --work-tree="~/rt/riak/" reset HEAD --hard
08:56:52.522 [debug] Running: git --git-dir="~/rt/riak/.git" --work-tree="~/rt/riak/" clean -fd
08:56:52.718 [notice] Running Test verify_build_cluster
08:56:52.719 [info] rt:set_backend(riak_kv_bitcask_backend)
08:56:52.719 [info] rtdev:set_backend(riak_kv_bitcask_backend)
08:56:52.719 [info] rtdev:update_app_config(all, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.720 [info] rtdev:update_app_config_file(~/rt/riak/current/dev/dev1/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.726 [info] rtdev:update_app_config_file(~/rt/riak/current/dev/dev2/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.731 [info] rtdev:update_app_config_file(~/rt/riak/current/dev/dev3/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.737 [info] rtdev:update_app_config_file(~/rt/riak/current/dev/dev4/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.743 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.2.0/dev/dev1/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.749 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.2.0/dev/dev2/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.755 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.2.0/dev/dev3/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.760 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.2.0/dev/dev4/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.766 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.1.4/dev/dev1/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.772 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.1.4/dev/dev2/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.780 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.1.4/dev/dev3/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.787 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.1.4/dev/dev4/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.794 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.0.3/dev/dev1/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.800 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.0.3/dev/dev2/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.805 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.0.3/dev/dev3/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.811 [info] rtdev:update_app_config_file(~/rt/riak/riak-1.0.3/dev/dev4/etc/app.config, [{riak_kv,[{storage_backend,riak_kv_bitcask_backend}]}])
08:56:52.850 [info] Deploying 3 nodes
08:56:52.850 [info] Riak path: "~/rt/riak"
08:56:52.887 [info] Running: ~/rt/riak/current/dev/dev1/bin/riak start
08:56:52.887 [info] Running: ~/rt/riak/current/dev/dev2/bin/riak start
08:56:52.887 [info] Running: ~/rt/riak/current/dev/dev3/bin/riak start
08:56:58.528 [debug] Supervisor inet_gethost_native_sup started undefined at pid <0.142.0>
08:56:58.528 [debug] Supervisor kernel_safe_sup started inet_gethost_native:start_link() at pid <0.141.0>
08:56:58.570 [info] Deployed nodes: ['dev1@127.0.0.1','dev2@127.0.0.1','dev3@127.0.0.1']
08:56:58.570 [info] Ensure each nodes 100% of it's own ring
08:56:58.572 [info] Join nodes together
08:56:58.601 [debug] [join] 'dev2@127.0.0.1' to ('dev1@127.0.0.1'): ok
08:56:58.646 [debug] [join] 'dev3@127.0.0.1' to ('dev1@127.0.0.1'): ok
08:56:58.646 [info] Wait until all nodes are ready and there are no pending changes
08:57:07.245 [info] Ensure each node owns a portion of the ring
08:57:07.249 [info] verify_build_cluster: PASS
08:57:07.249 [notice] verify_build_cluster Test Run Complete
'''
</pre>

As shown, the tests look similar to standard eunit tests and are
straightforward Erlang. The {@link rt} module provides the basic
commands to control Riak nodes, and abstracts everything away from
the underlying harness: devrel. The `rt' module
also provides a set of useful existing built-in operations. It is
expected that the `rt' module will be extended over time with more
reusable functions.

Writing additional checks and preconditions is the same as with an
eunit test. For example, `rt:owners_according_to(Node)' is just a
basic RPC call:

<pre class="erlang">
```
owners_according_to(Node) ->
    {ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []),
    Owners = [Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)],
    lists:usort(Owners).
'''
</pre>

And the `rt:wait_until_nodes_ready', which leverages the built-in `wait_until'
primitive:

<pre>
```
wait_until_nodes_ready(Nodes) ->
    [?assertEqual(ok, wait_until(Node, fun is_ready/1)) || Node <- Nodes],
    ok.

is_ready(Node) ->
    case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of
        {ok, Ring} ->
            lists:member(Node, riak_core_ring:ready_members(Ring));
        _ ->
            false
    end.
'''
</pre>

Erlang term based config files. The references `rtdev.config' file above:

```
%% Deps should include the path to compiled Riak libs
{rt_deps, ["/Users/jtuple/basho/riak/deps"]}.

%% Maximum time in milliseconds for wait_until to wait
{rt_max_wait_time, 180000}.

%% Delay between each retry in wait_until
{rt_retry_delay, 500}.

%% Use the devrel harness
{rt_harness, rtdev}.

%% Path to generated devrel of Riak that is git-versioned
{rtdev_path, "~/rt/riak"}.
'''

And to conclude, a slightly longer test that uses most of the built-in
primitives including starting and stopping Riak nodes:

<pre class="erlang">
```
-module(verify_claimant).
confirm() ->
    Nodes = build_cluster(3),
    [Node1, Node2, _Node3] = Nodes,

    %% Ensure all nodes believe node1 is the claimant
    lager:info("Ensure all nodes believe ~p is the claimant", [Node1]),
    [?assertEqual(Node1, claimant_according_to(Node)) || Node <- Nodes],

    %% Stop node1
    lager:info("Stop ~p", [Node1]),
    stop(Node1),
    ?assertEqual(ok, wait_until_unpingable(Node1)),

    %% Ensure all nodes still believe node1 is the claimant
    lager:info("Ensure all nodes still believe ~p is the claimant", [Node1]),
    Remaining = Nodes -- [Node1],
    [?assertEqual(Node1, claimant_according_to(Node)) || Node <- Remaining],

    %% Mark node1 as down and wait for ring convergence
    lager:info("Mark ~p as down", [Node1]),
    down(Node2, Node1),
    ?assertEqual(ok, wait_until_ring_converged(Remaining)),
    [?assertEqual(down, status_of_according_to(Node1, Node)) || Node <- Remaining],

    %% Ensure all nodes now believe node2 to be the claimant
    lager:info("Ensure all nodes now believe ~p is the claimant", [Node2]),
    [?assertEqual(Node2, claimant_according_to(Node)) || Node <- Remaining],

    %% Restart node1 and wait for ring convergence
    lager:info("Restart ~p and wait for ring convergence", [Node1]),
    start(Node1),
    ?assertEqual(ok, wait_until_nodes_ready([Node1])),
    ?assertEqual(ok, wait_until_ring_converged(Nodes)),

    %% Ensure node has rejoined and is no longer down
    lager:info("Ensure ~p has rejoined and is no longer down", [Node1]),
    [?assertEqual(valid, status_of_according_to(Node1, Node)) || Node <- Nodes],

    %% Ensure all nodes still believe node2 is the claimant
    lager:info("Ensure all nodes still believe ~p is the claimant", [Node2]),
    [?assertEqual(Node2, claimant_according_to(Node)) || Node <- Nodes],
    pass.
'''
</pre>

<h2>Installation and Usage</h2>
Checkout riak_test from Github and build: `http://github.com/basho/riak_test'

riak_test comes with two pre-written config files: `rtdev.config' for
devrel mode and `rtbe.config' for basho_expect mode. The devrel
operating mode comes built into riak_test, and can be used right away.

To use the basho_expect mode, you must checkout a version of basho_expect that
includes the basho_expect_thrift server mode, and make sure to run `make' from
within the `basho_expect/basho_expect_thrift' directory.

For both modes, ensure the `rt_deps' setting in the config file is accurate,
and adjust other settings as appropriate. The comments should be self explanatory.

To run riak_test in devrel mode, you must first generate a git-versioned devrel
directory and ensure the `rtdev_path' setting in `rtdev.config' is accurate.
To generate the versioned devrel, modify the following as necessary:
```
       riak$ make devrel
       riak$ mkdir ~/rt/riak
       riak$ cp -r dev ~/rt/riak/dev
       riak$ cd ~/rt/riak/dev
~/rt/riak/dev$ git init
~/rt/riak/dev$ git add .
~/rt/riak/dev$ git commit -m "initial"
'''

You can then launch tests from within the riak_test directory:
```
./riak_test -c rtdev -t verify_build_cluster
'''

riak_test currently assumes tests are written such that there is one
test per module and that the module contains a same named zero arity
function that implements the test. For example, the above calls
`verify_build_cluster:confirm()'.