riak_test/tests/pr_pw.erl

-module(pr_pw).

-behavior(riak_test).
-export([confirm/0]).

-include_lib("eunit/include/eunit.hrl").

confirm() ->
    application:start(inets),
    lager:info("Deploy some nodes"),
    Nodes = rt:build_cluster(4),

    %% calculate the preflist for foo/bar
    {ok, Ring} = rpc:call(hd(Nodes), riak_core_ring_manager, get_my_ring, []),
    UpNodes = rpc:call(hd(Nodes), riak_core_node_watcher, nodes, [riak_kv]),
    DocIdx = rpc:call(hd(Nodes), riak_core_util, chash_key, [{<<"foo">>,
                <<"bar">>}]),
    N = 3,
    Preflist2 = riak_core_apl:get_apl_ann(DocIdx, N, Ring, UpNodes),
    lager:info("Preflist is ~p", [Preflist2]),
    PLNodes = [Node || {{_Index, Node}, _Status} <- Preflist2],
    lager:info("Nodes in preflist ~p", [PLNodes]),
    [SafeNode] = Nodes -- PLNodes,
    lager:info("Node not involved in this preflist ~p", [SafeNode]),
    %% connect to the only node in the preflist we won't break, to avoid
    %% random put forwarding
    {ok, C} = riak:client_connect(hd(PLNodes)),
    NodeUrl = rt:http_url(hd(PLNodes)),
    UrlFun=fun(Key, Value, Params) ->
            lists:flatten(io_lib:format("~s/riak/~s/~s~s",
                    [NodeUrl, Key, Value, Params]))
    end,

    Obj = riak_object:new(<<"foo">>, <<"bar">>, <<42:32/integer>>),
    ?assertEqual(ok, C:put(Obj, [{pw, all}])),
    ?assertMatch({ok, _}, C:get(<<"foo">>, <<"bar">>, [{pr, all}])),

    %% check pr/pw can't be violated
    ?assertEqual({error, {pw_val_violation, evil}}, C:put(Obj, [{pw, evil}])),
    ?assertEqual({error, {pr_val_violation, evil}}, C:get(<<"foo">>, <<"bar">>,
            [{pr, evil}])),

    ?assertMatch({ok, {{_, 400, _}, _, "pr query parameter must be"++_}},
        httpc:request(get, {UrlFun(<<"foo">>, <<"bar">>, <<"?pr=evil">>), []}, [], [])),

    ?assertMatch({ok, {{_, 400, _}, _, "pw query parameter must be"++_}},
        httpc:request(put, {UrlFun(<<"foo">>, <<"bar">>,
                    <<"?pw=evil">>), [], "text/plain", <<42:32/integer>>}, [], [])),

    %% install an intercept to emulate a node that kernel paniced or
    %% something where it can take some time for the node_watcher to spot the
    %% downed node
    {{Index, Node}, _} = lists:last(Preflist2),
    make_intercepts_tab(Node, Index),
    rt_intercept:add(Node, {riak_kv_vnode,  [{{do_get,4}, drop_do_get},
                                                {{do_head, 4}, drop_do_head},
                                                {{do_put, 7}, drop_do_put}]}),
    lager:info("disabling do_get and do_head for index ~p on ~p", [Index, Node]),
    rt:log_to_nodes(Nodes, "disabling do_get and do_head for index ~p on ~p", [Index, Node]),
    timer:sleep(100),

    %% one vnode will never return, so we get timeouts
    ?assertEqual({error, timeout},
        C:get(<<"foo">>, <<"bar">>, [{pr, all}])),
    ?assertEqual({error, timeout}, C:put(Obj, [{pw, all}])),

    %% we can still meet quorum, though
    ?assertEqual(ok, C:put(Obj, [{pw, quorum}])),
    ?assertMatch({ok, _},
        C:get(<<"foo">>, <<"bar">>, [{pr, quorum}])),

    rt:stop_and_wait(Node),

    %% there's now a fallback in the preflist, so PR/PW won't be satisfied
    %% anymore
    ?assertEqual({error, {pr_val_unsatisfied, 3, 2}},
        C:get(<<"foo">>, <<"bar">>, [{pr, all}])),
    ?assertEqual({error, {pw_val_unsatisfied, 3, 2}}, C:put(Obj, [{pw, all}])),

    ?assertMatch({ok, {{_, 503, _}, _, "PR-value unsatisfied: 2/3\n"}},
        httpc:request(get, {UrlFun(<<"foo">>, <<"bar">>, <<"?pr=all">>), []}, [], [])),

    ?assertMatch({ok, {{_, 503, _}, _, "PW-value unsatisfied: 2/3\n"}},
        httpc:request(put, {UrlFun(<<"foo">>, <<"bar">>,
                    <<"?pw=all">>), [], "text/plain", <<42:32/integer>>}, [], [])),

    %% emulate another node failure
    {{Index2, Node2}, _} = lists:nth(2, Preflist2),
    make_intercepts_tab(Node2, Index2),
    rt_intercept:add(Node2, {riak_kv_vnode,  [{{do_get,4}, drop_do_get},
                                                {{do_head, 4}, drop_do_head},
                                                {{do_put, 7}, drop_do_put}]}),
    lager:info("disabling do_get, do_put and do_head for index ~p on ~p", [Index2, Node2]),
    rt:log_to_nodes(Nodes, "disabling do_get, do_put and do_head for index ~p on ~p", [Index2, Node2]),
    timer:sleep(100),

    %% can't even meet quorum now
    ?assertEqual({error, timeout},
        C:get(<<"foo">>, <<"bar">>, [{pr, quorum}])),
    ?assertEqual({error, timeout}, C:put(Obj, [{pw, quorum}])),

    %% restart the node
    rt:start_and_wait(Node),
    rt:wait_for_service(Node, riak_kv),

    %% we can make quorum again
    ?assertEqual(ok, C:put(Obj, [{pw, quorum}])),
    ?assertMatch({ok, _}, C:get(<<"foo">>, <<"bar">>, [{pr, quorum}])),
    %% intercepts still in force on second node, so we'll get timeouts
    ?assertEqual({error, timeout},
        C:get(<<"foo">>, <<"bar">>, [{pr, all}])),
    ?assertEqual({error, timeout}, C:put(Obj, [{pw, all}])),

    %% reboot the node
    rt:stop_and_wait(Node2),
    rt:start_and_wait(Node2),
    rt:wait_for_service(Node2, riak_kv),

    %% everything is happy again
    ?assertEqual(ok, C:put(Obj, [{pw, all}])),
    ?assertMatch({ok, _}, C:get(<<"foo">>, <<"bar">>, [{pr, all}])),

    %% make a vnode start to fail puts
    make_intercepts_tab(Node2, Index2),
    rt_intercept:add(Node2, {riak_kv_vnode,  [{{do_put, 7}, error_do_put}]}),
    lager:info("failing do_put for index ~p on ~p", [Index2, Node2]),
    rt:log_to_nodes(Nodes, "failing do_put for index ~p on ~p", [Index2, Node2]),
    timer:sleep(100),

    %% there's now a failing vnode in the preflist, so PW/DW won't be satisfied
    %% anymore
    ?assertEqual({error, {pw_val_unsatisfied, 3, 2}}, C:put(Obj, [{pw, all}])),
    ?assertEqual({error, {dw_val_unsatisfied, 3, 2}}, C:put(Obj, [{dw, all}])),

    ?assertMatch({ok, {{_, 503, _}, _, "PW-value unsatisfied: 2/3\n"}},
        httpc:request(put, {UrlFun(<<"foo">>, <<"bar">>,
                    <<"?pw=all">>), [], "text/plain", <<42:32/integer>>}, [], [])),
    ?assertMatch({ok, {{_, 503, _}, _, "DW-value unsatisfied: 2/3\n"}},
        httpc:request(put, {UrlFun(<<"foo">>, <<"bar">>,
                    <<"?dw=all">>), [], "text/plain", <<42:32/integer>>}, [], [])),
    pass.

make_intercepts_tab(Node, Partition) ->
    SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]),
    intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table,
                public, set, {heir, SupPid, {}}]]),
    true = rpc:call(Node, ets, insert, [intercepts_tab, {drop_do_get_partitions,
                [Partition]}]),
    true = rpc:call(Node, ets, insert, [intercepts_tab, {drop_do_head_partitions,
                [Partition]}]),
    true = rpc:call(Node, ets, insert, [intercepts_tab, {drop_do_put_partitions,
                [Partition]}]).