riak_test/tests/riak667_mixed.erl

%% -------------------------------------------------------------------
%%
%% Copyright (c) 2012 Basho Technologies, Inc.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -------------------------------------------------------------------
-module(riak667_mixed).
-behavior(riak_test).
-export([confirm/0]).
-include_lib("eunit/include/eunit.hrl").

-define(HARNESS, (rt_config:get(rt_harness))).
-define(TYPE, <<"maps">>).
-define(KEY, <<"cmeiklejohn">>).
-define(KEY2, <<"cmeik">>).
-define(BUCKET, {?TYPE, <<"testbucket">>}).

-define(CONF, [
               {riak_core,
                [{ring_creation_size, 8}]
               },
               {riak_kv,
                [{mdc_crdt_epoch, 1}]}
              ]).

confirm() ->
    rt:set_advanced_conf(all, ?CONF),

    %% Configure cluster.
    TestMetaData = riak_test_runner:metadata(),
    OldVsn = proplists:get_value(upgrade_version, TestMetaData, "2.0.2"),
    Nodes = [Node1, Node2] = rt:build_cluster([OldVsn, OldVsn]),

    CurrentVer = rt:get_version(),

    lager:info("mdc_crdt_epoch? ~p", [rpc:multicall(Nodes, application, get_env, [riak_kv, mdc_crdt_epoch])]),

    %% Create PB connection.
    Pid = rt:pbc(Node1),
    riakc_pb_socket:set_options(Pid, [queue_if_disconnected]),

    %% Create bucket type for maps.
    rt:create_and_activate_bucket_type(Node1, ?TYPE, [{datatype, map}]),

    %% Write some sample data.
    Map = riakc_map:update(
            {<<"names">>, set},
            fun(R) ->
                    riakc_set:add_element(<<"Original">>, R)
            end, riakc_map:new()),
    Map2 = riakc_map:update({<<"profile">>, map},
                            fun(M) ->
                                    riakc_map:update(
                                      {<<"name">>, register},
                                      fun(R) ->
                                              riakc_register:set(<<"Bob">>, R)
                                      end, M)
                            end, Map),

    ok = riakc_pb_socket:update_type(
            Pid,
            ?BUCKET,
            ?KEY,
            riakc_map:to_op(Map2)),

    %% Upgrade one node.
    upgrade(Node2, "2.0.4"),

    lager:notice("running mixed 2.0.2 and 2.0.4"),

    %% Create PB connection.
    Pid2 = rt:pbc(Node2),
    riakc_pb_socket:set_options(Pid2, [queue_if_disconnected]),

    %% Read value.
    ?assertMatch({error, <<"Error processing incoming message: error:{badrecord,dict}", _/binary>>},
                 riakc_pb_socket:fetch_type(Pid2, ?BUCKET, ?KEY)),

    lager:notice("Can't read a 2.0.2 map from 2.0.4 node"),

    %% Write some 2.0.4 data.
    Oh4Map = riakc_map:update(
            {<<"names">>, set},
            fun(R) ->
                    riakc_set:add_element(<<"Original">>, R)
            end, riakc_map:new()),
    Oh4Map2 = riakc_map:update({<<"profile">>, map},
                            fun(M) ->
                                    riakc_map:update(
                                      {"name", register},
                                      fun(R) ->
                                              riakc_register:set(<<"Bob">>, R)
                                      end, M)
                            end, Oh4Map),

    ok = riakc_pb_socket:update_type(
            Pid2,
            ?BUCKET,
            ?KEY2,
            riakc_map:to_op(Oh4Map2)),

    lager:notice("Created a 2.0.4 map"),

    %% and read 2.0.4 data?? Nope, dict is not an orddict
    ?assertMatch({error,<<"Error processing incoming message: error:function_clause:[{orddict,fold", _/binary>>},
                 riakc_pb_socket:fetch_type(Pid, ?BUCKET, ?KEY2)),

    lager:notice("Can't read 2.0.4 map from 2.0.2 node"),

    %% upgrade 2.0.4 to 2.0.5
    riakc_pb_socket:stop(Pid2),
    upgrade(Node2, current),

    lager:notice("running mixed 2.0.2 and ~s", [CurrentVer]),

    %% Create PB connection.
    Pid3 = rt:pbc(Node2),
    riakc_pb_socket:set_options(Pid3, [queue_if_disconnected]),

    %% read both maps
    {ok, K1O} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY),
    {ok, K2O} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY2),

    lager:notice("2.0.2 map ~p", [K1O]),
    lager:notice("2.0.4 map ~p", [K2O]),

    %% update 2.0.2 map on new node ?KEY Pid3
    K1OU = riakc_map:update({<<"profile">>, map},
                            fun(M) ->
                                    riakc_map:update(
                                      {<<"name">>, register},
                                      fun(R) ->
                                              riakc_register:set(<<"Rita">>, R)
                                      end, M)
                            end, K1O),

    ok = riakc_pb_socket:update_type(Pid3, ?BUCKET, ?KEY, riakc_map:to_op(K1OU)),
    lager:notice("Updated 2.0.2 map on ~s", [CurrentVer]),

    %% read 2.0.2 map from 2.0.2 node ?KEY Pid
    {ok, K1OR} = riakc_pb_socket:fetch_type(Pid, ?BUCKET, ?KEY),
    lager:notice("Read 2.0.2 map from 2.0.2 node: ~p", [K1OR]),

    ?assertEqual(<<"Rita">>, orddict:fetch({<<"name">>, register},
                                           riakc_map:fetch({<<"profile">>, map}, K1OR))),

    %% update 2.0.2 map on old node ?KEY Pid
    K1O2 = riakc_map:update({<<"profile">>, map},
                            fun(M) ->
                                    riakc_map:update(
                                      {<<"name">>, register},
                                      fun(R) ->
                                              riakc_register:set(<<"Sue">>, R)
                                      end, M)
                            end, K1OR),

    ok = riakc_pb_socket:update_type(Pid, ?BUCKET, ?KEY, riakc_map:to_op(K1O2)),
    lager:notice("Updated 2.0.2 map on 2.0.2 node"),

    %% read it from 2.0.5 node ?KEY Pid3
    {ok, K1OC} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY),
    lager:notice("Read 2.0.2 map from ~s node: ~p", [CurrentVer, K1OC]),

    ?assertEqual(<<"Sue">>, orddict:fetch({<<"name">>, register},
                                          riakc_map:fetch({<<"profile">>, map}, K1OC))),

    %% update 2.0.4 map node ?KEY2 Pid3
    K2OU = riakc_map:update({<<"people">>, set},
                            fun(S) ->
                                    riakc_set:add_element(<<"Joan">>, S)
                            end, riakc_map:new()),

    ok = riakc_pb_socket:update_type(Pid3, ?BUCKET, ?KEY2, riakc_map:to_op(K2OU)),
    lager:notice("Updated 2.0.4 map on ~s node", [CurrentVer]),
    %% upgrade 2.0.2 node

    riakc_pb_socket:stop(Pid),
    upgrade(Node1, current),
    lager:notice("Upgraded 2.0.2 node to ~s", [CurrentVer]),

    %% read and write maps
    Pid4 = rt:pbc(Node1),

    {ok, K1N1} = riakc_pb_socket:fetch_type(Pid4, ?BUCKET, ?KEY),
    {ok, K1N2} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY),
    ?assertEqual(K1N1, K1N2),

    {ok, K2N1} = riakc_pb_socket:fetch_type(Pid4, ?BUCKET, ?KEY2),
    {ok, K2N2} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY2),
    ?assertEqual(K2N1, K2N2),
    lager:notice("Maps fetched from both nodes are same K1:~p K2:~p", [K1N1, K2N1]),

    K1M = riakc_map:update({<<"people">>, set},
                           fun(S) -> riakc_set:add_element(<<"Roger">>, S) end,
                           K1N1),
    ok = riakc_pb_socket:update_type(Pid3, ?BUCKET, ?KEY, riakc_map:to_op(K1M)),

    K2M = riakc_map:update({<<"people">>, set},
                           fun(S) -> riakc_set:add_element(<<"Don">>, S) end,
                           K2N1),
    ok = riakc_pb_socket:update_type(Pid4, ?BUCKET, ?KEY2, riakc_map:to_op(K2M)),
    %% (how???) check format is still v1 (maybe get raw kv object and inspect contents using riak_kv_crdt??

    {ok, Robj1} = riakc_pb_socket:get(Pid3, ?BUCKET, ?KEY),
    ?assert(map_contents_are_lists(Robj1)),

    {ok, Robj2} = riakc_pb_socket:get(Pid4, ?BUCKET, ?KEY2),

    lager:info("mdc_crdt_epoch? ~p", [rpc:multicall(Nodes, application, get_env, [riak_kv, mdc_crdt_epoch])]),

    ?assert(map_contents_are_lists(Robj2)),
    %% unset env var
    rpc:multicall(Nodes, application, set_env, [riak_kv, mdc_crdt_epoch, false]),

    lager:info("mdc_crdt_epoch? ~p", [rpc:multicall(Nodes, application, get_env, [riak_kv, mdc_crdt_epoch])]),

    %% read and write maps
    {ok, Up1N1} = riakc_pb_socket:fetch_type(Pid4, ?BUCKET, ?KEY),
    {ok, Up1N2} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY),
    ?assertEqual(Up1N1, Up1N2),

    {ok, Up2N1} = riakc_pb_socket:fetch_type(Pid4, ?BUCKET, ?KEY2),
    {ok, Up2N2} = riakc_pb_socket:fetch_type(Pid3, ?BUCKET, ?KEY2),
    ?assertEqual(Up2N1, Up2N2),
    lager:notice("Maps fetched from both nodes are same K1:~p K2:~p", [Up1N1, Up2N1]),

    Up1M = riakc_map:update({<<"people">>, set},
                           fun(S) -> riakc_set:add_element(<<"Betty">>, S) end,
                           Up1N1),
    ok = riakc_pb_socket:update_type(Pid3, ?BUCKET, ?KEY, riakc_map:to_op(Up1M)),

    Up2M = riakc_map:update({<<"people">>, set},
                           fun(S) -> riakc_set:add_element(<<"Burt">>, S) end,
                           Up2N1),
    ok = riakc_pb_socket:update_type(Pid4, ?BUCKET, ?KEY2, riakc_map:to_op(Up2M)),

    %% (how??? see above?) check ondisk format is now v2
    {ok, UpObj1} = riakc_pb_socket:get(Pid3, ?BUCKET, ?KEY),
    ?assert(map_contents_are_dicts(UpObj1)),

    {ok, UpObj2} = riakc_pb_socket:get(Pid4, ?BUCKET, ?KEY2),
    ?assert(map_contents_are_dicts(UpObj2)),

    %% Stop PB connection.
    riakc_pb_socket:stop(Pid3),
    riakc_pb_socket:stop(Pid4),

    pass.

upgrade(Node, NewVsn) ->
    lager:notice("Upgrading ~p to ~p", [Node, NewVsn]),
    rt:upgrade(Node, NewVsn),
    rt:wait_for_service(Node, riak_kv),
    ok.

map_contents_are_lists(RObj) ->
    [{_MD, V}] = riakc_obj:get_contents(RObj),
    {riak_dt_map, {_Clock, Entries, Deferred}} = map_from_binary(V),
    lager:info("Top-level map: ~p || ~p", [Entries, Deferred]),
    is_list(Entries) andalso is_list(Deferred) andalso nested_are_lists(Entries).

nested_are_lists(Entries) ->
    %% This is ugly because it reaches into the guts of the data
    %% structure's internal format.
    lists:all(fun({{_, riak_dt_orswot}, {CRDTs, Tombstone}}) ->
                         lists:all(fun({_Dot, Set}) -> set_is_list(Set) end, CRDTs)
                             andalso set_is_list(Tombstone);
                 ({{_, riak_dt_map}, {CRDTs, Tombstone}}) ->
                         lists:all(fun({_Dot, Map}) -> map_is_list(Map) end, CRDTs)
                             andalso map_is_list(Tombstone);
                 (_) ->
                      true
              end, Entries).

set_is_list({_Clock, Entries, Deferred}) ->
    is_list(Entries) andalso is_list(Deferred).

map_is_list({_Clock, Entries, Deferred}) ->
    is_list(Deferred) andalso nested_are_lists(Entries).

map_contents_are_dicts(RObj) ->
    [{_MD, V}] = riakc_obj:get_contents(RObj),
    {riak_dt_map, {_Clock, Entries, Deferred}} = map_from_binary(V),
    lager:info("Top-level map: ~p || ~p", [Entries, Deferred]),
    is_dict(Entries) andalso is_dict(Deferred) andalso nested_are_dicts(Entries).

is_dict(V) ->
    is_tuple(V) andalso dict == element(1, V).

nested_are_dicts(Entries) ->
    %% This is ugly because it reaches into the guts of the data
    %% structure's internal format.
    lists:all(fun({{_, riak_dt_orswot}, {CRDTs, Tombstone}}) ->
                      is_dict(CRDTs) andalso
                          set_is_dict(Tombstone) andalso
                          dict:fold(fun(_K, Set, Acc) ->
                                            set_is_dict(Set) andalso Acc
                                    end, true, CRDTs);
                 ({{_, riak_dt_map}, {CRDTs, Tombstone}}) ->
                      is_dict(CRDTs) andalso map_is_dict(Tombstone) andalso
                          dict:fold(fun(_K, Map, Acc) ->
                                            map_is_dict(Map) andalso Acc
                                    end, true, CRDTs);
                 (_) ->
                      true
              end, dict:to_list(Entries)).

set_is_dict({_Clock, Entries, Deferred}) ->
    is_dict(Entries) andalso is_dict(Deferred).

map_is_dict({_Clock, Entries, Deferred}) ->
    is_dict(Deferred) andalso nested_are_dicts(Entries).

%% Somewhat copy-pasta from riak_kv_crdt
%% NB:?TAG is 69 in riak_kv_crdt, version is 2
%%    ?TAG is 77 in riak_dt_types.hrl, version is 1 or 2
map_from_binary(<<69:8, 2:8, TypeLen:32/integer, Type:TypeLen/binary, 77:8, MapVer:8,
                  CRDTBin/binary>>) ->
    lager:notice("Deserialized Map: ~s v~p", [Type, MapVer]),
    Mod = binary_to_atom(Type, latin1),
    {Mod, riak_dt:from_binary(CRDTBin)}.