From fad85d43594b51b489770d4fb1e641b6305c04fc Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Tue, 28 Feb 2017 17:24:50 -0500 Subject: [PATCH 1/5] Add EQC test for list group keys --- tests/kv_list_group_keys_eqc.erl | 190 +++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/kv_list_group_keys_eqc.erl diff --git a/tests/kv_list_group_keys_eqc.erl b/tests/kv_list_group_keys_eqc.erl new file mode 100644 index 00000000..954ce1b6 --- /dev/null +++ b/tests/kv_list_group_keys_eqc.erl @@ -0,0 +1,190 @@ +-module(kv_list_group_keys_eqc). + +-ifdef(EQC). + +-compile(export_all). +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-behaviour(riak_test). +-export([confirm/0]). + +-define(CONFIG, [{riak_kv, [{storage_backend, riak_kv_eleveldb_backend}]}]). +-define(CLUSTER_SIZE, 5). +-define(PHOTOS_PREFIX, <<"photos/2017/">>). +-define(MONTHS, [<<"January">>, <<"February">>, <<"March">>, <<"April">>]). + +-record(group_keys_result, { + keys = [] :: list(), + common_prefixes = [] :: list() + }). + +confirm() -> + Cluster = setup_cluster(?CLUSTER_SIZE), + put_objects(Cluster), + quickcheck(?MODULE:prop_group_keys_basic(Cluster)), + quickcheck(?MODULE:prop_group_keys_prefix(Cluster)), + quickcheck(?MODULE:prop_group_keys_delimiter(Cluster)), + quickcheck(?MODULE:prop_group_keys_prefix_delimiter(Cluster)), + rt:clean_cluster(Cluster), + pass. + +%% ==================================================================== +%% EQC generators +%% ==================================================================== +gen_max_keys() -> + choose(1, length(all_keys())). + +%% ==================================================================== +%% EQC Properties +%% ==================================================================== + +quickcheck(Property) -> + ?assert(eqc:quickcheck(eqc:numtests(500, Property))). + +prop_group_keys_basic(Cluster) -> + Expected = #group_keys_result{keys = lists:sort(all_keys())}, + ?FORALL(Bucket, oneof(buckets()), + ?FORALL(MaxKeys, gen_max_keys(), + Expected == collect_group_keys( + Cluster, + Bucket, + make_group_params([{max_keys, MaxKeys}])))). + +prop_group_keys_prefix(Cluster) -> + Prefix = ?PHOTOS_PREFIX, + GroupParams = make_group_params([{prefix, Prefix}]), + Expected = #group_keys_result{ + keys = lists:sort(lists:filter(fun(Key) -> + is_prefix(Prefix, Key) + end, + all_keys())) + }, + ?FORALL(Bucket, oneof(buckets()), + ?FORALL(MaxKeys, gen_max_keys(), + begin + GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + Expected == collect_group_keys(Cluster, Bucket, GroupParams1) + end)). + +prop_group_keys_delimiter(Cluster) -> + Delimiter = <<"/">>, + GroupParams = make_group_params([{delimiter, Delimiter}]), + Expected = #group_keys_result{ + keys = [<<"sample.jpg">>], + common_prefixes = [<<"photos/">>] + }, + ?FORALL(Bucket, oneof(buckets()), + ?FORALL(MaxKeys, gen_max_keys(), + begin + GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + Expected == collect_group_keys(Cluster, Bucket, GroupParams1) + end)). + +prop_group_keys_prefix_delimiter(Cluster) -> + Prefix = ?PHOTOS_PREFIX, + Delimiter = <<"/">>, + GroupParams = make_group_params([{prefix, Prefix}, {delimiter, Delimiter}]), + Expected = #group_keys_result{ + keys = lists:sort([<>, + <>]), + common_prefixes = lists:sort([<> + || Month <- ?MONTHS]) + }, + ?FORALL(Bucket, oneof(buckets()), + ?FORALL(MaxKeys, gen_max_keys(), + begin + GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + Expected == collect_group_keys(Cluster, Bucket, GroupParams1) + end)). + +%% ==================================================================== +%% Helpers +%% ==================================================================== +setup_cluster(NumNodes) -> + Nodes = rt:build_cluster(NumNodes, ?CONFIG), + ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes)), + ?assertEqual(ok, rt:wait_until_transfers_complete(Nodes)), + Node1 = hd(Nodes), + [begin + rt:create_and_activate_bucket_type(Node1, BucketType, [{n_val, NVal}]), + rt:wait_until_bucket_type_status(BucketType, active, Nodes), + rt:wait_until_bucket_type_visible(Nodes, BucketType) + end || {BucketType, NVal} <- bucket_types()], + Nodes. + +bucket_types() -> + [{<<"n_val_1">>, 1}, + {<<"n_val_2">>, 2}, + {<<"n_val_3">>, 3}, + {<<"n_val_4">>, 4}, + {<<"n_val_5">>, 5}]. + +buckets() -> + [{BucketType, BucketType} || {BucketType, _} <- bucket_types()]. + +all_keys() -> + [<<"sample.jpg">>, + <>, + <>] ++ + [<> + || Month <- ?MONTHS, N <- [$1, $2, $3, $4, $5]]. + +put_objects(Cluster) -> + lists:foreach(fun({BucketType, _}) -> + put_objects(Cluster, {BucketType, BucketType}) + end, + bucket_types()). + +put_objects(Cluster, Bucket) -> + Node = rt:select_random(Cluster), + {ok, Client} = riak:client_connect(Node), + lists:foreach(fun(Key) -> + put_object(Client, Bucket, Key) + end, + all_keys()). + +put_object(Client, Bucket, Key) -> + Obj = riak_object:new(Bucket, Key, Key), + ok = Client:put(Obj). + +make_group_params(PropList) -> + riak_kv_group_keys:to_group_params(PropList). + +is_prefix(B1, B2) when is_binary(B1), is_binary(B2) -> + binary:longest_common_prefix([B1, B2]) == size(B1); +is_prefix(_B1, _B2) -> + false. + +collect_group_keys(Cluster, Bucket, GroupParams) -> + collect_group_keys(Cluster, Bucket, GroupParams, #group_keys_result{}). + +collect_group_keys(Cluster, Bucket, GroupParams, Acc) -> + Response = list_group_keys(Cluster, Bucket, GroupParams), + NewAcc = accumulate_group_keys(Response, Acc), + case riak_kv_group_keys_response:get_next_continuation_token(Response) of + undefined -> + NewAcc; + ContinuationToken -> + NewGroupParams = set_continuation_token(GroupParams, ContinuationToken), + collect_group_keys(Cluster, Bucket, NewGroupParams, NewAcc) + end. + +list_group_keys(Cluster, Bucket, GroupParams) -> + Node = rt:select_random(Cluster), + {ok, Client} = riak:client_connect(Node), + {ok, Response} = Client:list_group_keys(Bucket, GroupParams, infinity), + Response. + +accumulate_group_keys(GroupKeysResponse, Acc) -> + Metadatas = riak_kv_group_keys_response:get_metadatas(GroupKeysResponse), + CommonPrefixes = riak_kv_group_keys_response:get_common_prefixes(GroupKeysResponse), + Acc#group_keys_result{ + keys = Acc#group_keys_result.keys ++ [Key || {Key, _Meta} <- Metadatas], + common_prefixes = Acc#group_keys_result.common_prefixes ++ CommonPrefixes + }. + +set_continuation_token(GroupParams, ContinuationToken) -> + riak_kv_group_keys:set_continuation_token(GroupParams, ContinuationToken). + +-endif. % EQC From cbc0fe86ce286e43dabaec03165b73faa35f3d37 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Thu, 2 Mar 2017 13:23:56 -0500 Subject: [PATCH 2/5] Load more test data and collect statistics --- tests/kv_list_group_keys_eqc.erl | 66 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/tests/kv_list_group_keys_eqc.erl b/tests/kv_list_group_keys_eqc.erl index 954ce1b6..53c248b3 100644 --- a/tests/kv_list_group_keys_eqc.erl +++ b/tests/kv_list_group_keys_eqc.erl @@ -9,10 +9,15 @@ -behaviour(riak_test). -export([confirm/0]). --define(CONFIG, [{riak_kv, [{storage_backend, riak_kv_eleveldb_backend}]}]). +-define(CONFIG, + [{riak_core, [{handoff_concurrency, 10}, + {vnode_management_timer, 1000}]}, + {riak_kv, [{storage_backend, riak_kv_eleveldb_backend}]}]). -define(CLUSTER_SIZE, 5). -define(PHOTOS_PREFIX, <<"photos/2017/">>). --define(MONTHS, [<<"January">>, <<"February">>, <<"March">>, <<"April">>]). +-define(MONTHS, [<<"January">>, <<"February">>, <<"March">>, <<"April">>, + <<"May">>, <<"June">>, <<"July">>, <<"August">>, + <<"September">>, <<"October">>, <<"November">>, <<"December">>]). -record(group_keys_result, { keys = [] :: list(), @@ -33,7 +38,8 @@ confirm() -> %% EQC generators %% ==================================================================== gen_max_keys() -> - choose(1, length(all_keys())). + NumKeys = length(all_keys()), + choose(erlang:max(1, NumKeys div 10), NumKeys). %% ==================================================================== %% EQC Properties @@ -42,14 +48,19 @@ gen_max_keys() -> quickcheck(Property) -> ?assert(eqc:quickcheck(eqc:numtests(500, Property))). +forall_buckets_and_max_keys(Property) -> + ?FORALL(Bucket, oneof(buckets()), + collect(Bucket, + ?FORALL(MaxKeys, gen_max_keys(), + collect(MaxKeys, Property(Bucket, MaxKeys))))). + prop_group_keys_basic(Cluster) -> Expected = #group_keys_result{keys = lists:sort(all_keys())}, - ?FORALL(Bucket, oneof(buckets()), - ?FORALL(MaxKeys, gen_max_keys(), - Expected == collect_group_keys( - Cluster, - Bucket, - make_group_params([{max_keys, MaxKeys}])))). + forall_buckets_and_max_keys( + fun(Bucket, MaxKeys) -> + GroupParams = make_group_params([{max_keys, MaxKeys}]), + Expected == collect_group_keys(Cluster, Bucket, GroupParams) + end). prop_group_keys_prefix(Cluster) -> Prefix = ?PHOTOS_PREFIX, @@ -60,12 +71,11 @@ prop_group_keys_prefix(Cluster) -> end, all_keys())) }, - ?FORALL(Bucket, oneof(buckets()), - ?FORALL(MaxKeys, gen_max_keys(), - begin - GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), - Expected == collect_group_keys(Cluster, Bucket, GroupParams1) - end)). + forall_buckets_and_max_keys( + fun(Bucket, MaxKeys) -> + GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + Expected == collect_group_keys(Cluster, Bucket, GroupParams1) + end). prop_group_keys_delimiter(Cluster) -> Delimiter = <<"/">>, @@ -74,12 +84,11 @@ prop_group_keys_delimiter(Cluster) -> keys = [<<"sample.jpg">>], common_prefixes = [<<"photos/">>] }, - ?FORALL(Bucket, oneof(buckets()), - ?FORALL(MaxKeys, gen_max_keys(), - begin - GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), - Expected == collect_group_keys(Cluster, Bucket, GroupParams1) - end)). + forall_buckets_and_max_keys( + fun(Bucket, MaxKeys) -> + GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + Expected == collect_group_keys(Cluster, Bucket, GroupParams1) + end). prop_group_keys_prefix_delimiter(Cluster) -> Prefix = ?PHOTOS_PREFIX, @@ -91,12 +100,11 @@ prop_group_keys_prefix_delimiter(Cluster) -> common_prefixes = lists:sort([<> || Month <- ?MONTHS]) }, - ?FORALL(Bucket, oneof(buckets()), - ?FORALL(MaxKeys, gen_max_keys(), - begin - GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), - Expected == collect_group_keys(Cluster, Bucket, GroupParams1) - end)). + forall_buckets_and_max_keys( + fun(Bucket, MaxKeys) -> + GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + Expected == collect_group_keys(Cluster, Bucket, GroupParams1) + end). %% ==================================================================== %% Helpers @@ -127,8 +135,8 @@ all_keys() -> [<<"sample.jpg">>, <>, <>] ++ - [<> - || Month <- ?MONTHS, N <- [$1, $2, $3, $4, $5]]. + [<> + || Month <- ?MONTHS, N <- lists:map(fun erlang:integer_to_binary/1, lists:seq(1, 99))]. put_objects(Cluster) -> lists:foreach(fun({BucketType, _}) -> From fdbf97f0e272e0f052c7280b480297954426f274 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Thu, 2 Mar 2017 14:10:44 -0500 Subject: [PATCH 3/5] Set timeout for list_group_keys to 5 seconds --- tests/kv_list_group_keys_eqc.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kv_list_group_keys_eqc.erl b/tests/kv_list_group_keys_eqc.erl index 53c248b3..04f4eb10 100644 --- a/tests/kv_list_group_keys_eqc.erl +++ b/tests/kv_list_group_keys_eqc.erl @@ -181,7 +181,7 @@ collect_group_keys(Cluster, Bucket, GroupParams, Acc) -> list_group_keys(Cluster, Bucket, GroupParams) -> Node = rt:select_random(Cluster), {ok, Client} = riak:client_connect(Node), - {ok, Response} = Client:list_group_keys(Bucket, GroupParams, infinity), + {ok, Response} = Client:list_group_keys(Bucket, GroupParams, 5000), Response. accumulate_group_keys(GroupKeysResponse, Acc) -> From d15d2a6bd04031b695e952732fee740f2e061cfb Mon Sep 17 00:00:00 2001 From: Doug Rohrer and Torben Hoffmann Date: Mon, 10 Apr 2017 08:23:33 -0400 Subject: [PATCH 4/5] Cleanup after refactor in `riak_kv` to "riak_kv_group_list". --- tests/kv_list_group_keys_eqc.erl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/kv_list_group_keys_eqc.erl b/tests/kv_list_group_keys_eqc.erl index 04f4eb10..899c864f 100644 --- a/tests/kv_list_group_keys_eqc.erl +++ b/tests/kv_list_group_keys_eqc.erl @@ -73,7 +73,7 @@ prop_group_keys_prefix(Cluster) -> }, forall_buckets_and_max_keys( fun(Bucket, MaxKeys) -> - GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + GroupParams1 = riak_kv_group_list:set_max_keys(GroupParams, MaxKeys), Expected == collect_group_keys(Cluster, Bucket, GroupParams1) end). @@ -86,7 +86,7 @@ prop_group_keys_delimiter(Cluster) -> }, forall_buckets_and_max_keys( fun(Bucket, MaxKeys) -> - GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + GroupParams1 = riak_kv_group_list:set_max_keys(GroupParams, MaxKeys), Expected == collect_group_keys(Cluster, Bucket, GroupParams1) end). @@ -102,7 +102,7 @@ prop_group_keys_prefix_delimiter(Cluster) -> }, forall_buckets_and_max_keys( fun(Bucket, MaxKeys) -> - GroupParams1 = riak_kv_group_keys:set_max_keys(GroupParams, MaxKeys), + GroupParams1 = riak_kv_group_list:set_max_keys(GroupParams, MaxKeys), Expected == collect_group_keys(Cluster, Bucket, GroupParams1) end). @@ -157,7 +157,7 @@ put_object(Client, Bucket, Key) -> ok = Client:put(Obj). make_group_params(PropList) -> - riak_kv_group_keys:to_group_params(PropList). + riak_kv_group_list:to_group_params(PropList). is_prefix(B1, B2) when is_binary(B1), is_binary(B2) -> binary:longest_common_prefix([B1, B2]) == size(B1); @@ -168,9 +168,9 @@ collect_group_keys(Cluster, Bucket, GroupParams) -> collect_group_keys(Cluster, Bucket, GroupParams, #group_keys_result{}). collect_group_keys(Cluster, Bucket, GroupParams, Acc) -> - Response = list_group_keys(Cluster, Bucket, GroupParams), + Response = list_group(Cluster, Bucket, GroupParams), NewAcc = accumulate_group_keys(Response, Acc), - case riak_kv_group_keys_response:get_next_continuation_token(Response) of + case riak_kv_group_list_response:get_next_continuation_token(Response) of undefined -> NewAcc; ContinuationToken -> @@ -178,21 +178,21 @@ collect_group_keys(Cluster, Bucket, GroupParams, Acc) -> collect_group_keys(Cluster, Bucket, NewGroupParams, NewAcc) end. -list_group_keys(Cluster, Bucket, GroupParams) -> +list_group(Cluster, Bucket, GroupParams) -> Node = rt:select_random(Cluster), {ok, Client} = riak:client_connect(Node), - {ok, Response} = Client:list_group_keys(Bucket, GroupParams, 5000), + {ok, Response} = Client:list_group(Bucket, GroupParams, 5000), Response. accumulate_group_keys(GroupKeysResponse, Acc) -> - Metadatas = riak_kv_group_keys_response:get_metadatas(GroupKeysResponse), - CommonPrefixes = riak_kv_group_keys_response:get_common_prefixes(GroupKeysResponse), + Metadatas = riak_kv_group_list_response:get_metadatas(GroupKeysResponse), + CommonPrefixes = riak_kv_group_list_response:get_common_prefixes(GroupKeysResponse), Acc#group_keys_result{ keys = Acc#group_keys_result.keys ++ [Key || {Key, _Meta} <- Metadatas], common_prefixes = Acc#group_keys_result.common_prefixes ++ CommonPrefixes }. set_continuation_token(GroupParams, ContinuationToken) -> - riak_kv_group_keys:set_continuation_token(GroupParams, ContinuationToken). + riak_kv_group_list:set_continuation_token(GroupParams, ContinuationToken). -endif. % EQC From a8f1a89a45c03bec7df4b995b02b2cbbe2102e71 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Mon, 10 Apr 2017 15:43:02 -0400 Subject: [PATCH 5/5] No need to clean cluster at end of test --- tests/kv_list_group_keys_eqc.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kv_list_group_keys_eqc.erl b/tests/kv_list_group_keys_eqc.erl index 04f4eb10..5c7bc3e0 100644 --- a/tests/kv_list_group_keys_eqc.erl +++ b/tests/kv_list_group_keys_eqc.erl @@ -31,7 +31,6 @@ confirm() -> quickcheck(?MODULE:prop_group_keys_prefix(Cluster)), quickcheck(?MODULE:prop_group_keys_delimiter(Cluster)), quickcheck(?MODULE:prop_group_keys_prefix_delimiter(Cluster)), - rt:clean_cluster(Cluster), pass. %% ====================================================================