From a8e667eab8cc5bdd3cfe8b8b08aecfc194d3057d Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Sat, 20 Apr 2013 18:22:34 +0100 Subject: [PATCH 1/6] 2i pagination tests --- tests/secondary_index_tests.erl | 129 +++++++++++++++++++++++++++++--- tests/verify_2i_limit.erl | 63 ++++++++++++++++ tests/verify_2i_returnterms.erl | 78 +++++++++++++++++++ tests/verify_2i_stream.erl | 68 +++++++++++++++++ 4 files changed, 328 insertions(+), 10 deletions(-) create mode 100644 tests/verify_2i_limit.erl create mode 100644 tests/verify_2i_returnterms.erl create mode 100644 tests/verify_2i_stream.erl diff --git a/tests/secondary_index_tests.erl b/tests/secondary_index_tests.erl index 6bc055ed..f94335cc 100644 --- a/tests/secondary_index_tests.erl +++ b/tests/secondary_index_tests.erl @@ -20,6 +20,9 @@ -module(secondary_index_tests). -behavior(riak_test). -export([confirm/0]). +-export([put_an_object/2, put_an_object/4, int_to_key/1, + stream_pb/2, stream_pb/3, pb_query/3, http_query/2, + http_query/3, http_stream/3, int_to_field1_bin/1]). -include_lib("eunit/include/eunit.hrl"). -define(BUCKET, <<"2ibucket">>). @@ -65,16 +68,6 @@ confirm() -> TestIdxVal), pass. -put_an_object(Pid, N) -> - lager:info("Putting object ~p", [N]), - Indexes = [{"field1_bin", list_to_binary(io_lib:format("val~p", [N]))}, - {"field2_int", N}], - MetaData = dict:from_list([{<<"index">>, Indexes}]), - Robj0 = riakc_obj:new(?BUCKET, list_to_binary(io_lib:format("obj~p", [N]))), - Robj1 = riakc_obj:update_value(Robj0, io_lib:format("data~p", [N])), - Robj2 = riakc_obj:update_metadata(Robj1, MetaData), - riakc_pb_socket:put(Pid, Robj2). - assertExactQuery(Pid, Expected, Index, Value) -> lager:info("Searching Index ~p for ~p", [Index, Value]), @@ -91,3 +84,119 @@ assertRangeQuery(Pid, Expected, Index, StartValue, EndValue) -> lager:info("Expected: ~p", [Expected]), lager:info("Actual : ~p", [ActualKeys]), ?assertEqual(Expected, ActualKeys). + +%% general 2i utility +put_an_object(Pid, N) -> + Key = int_to_key(N), + BinIndex = int_to_field1_bin(N), + put_an_object(Pid, Key, N, BinIndex). + +put_an_object(Pid, Key, IntIndex, BinIndex) -> + lager:info("Putting object ~p", [Key]), + Indexes = [{"field1_bin", BinIndex}, + {"field2_int", IntIndex}], + MetaData = dict:from_list([{<<"index">>, Indexes}]), + Robj0 = riakc_obj:new(?BUCKET, Key), + Robj1 = riakc_obj:update_value(Robj0, io_lib:format("data~p", [IntIndex])), + Robj2 = riakc_obj:update_metadata(Robj1, MetaData), + riakc_pb_socket:put(Pid, Robj2). + +int_to_key(N) -> + list_to_binary(io_lib:format("obj~p", [N])). + +int_to_field1_bin(N) -> + list_to_binary(io_lib:format("val~p", [N])). + +stream_pb(Pid, Q) -> + pb_query(Pid, Q, [stream]), + stream_loop(). + +stream_pb(Pid, Q, Opts) -> + pb_query(Pid, Q, [stream|Opts]), + stream_loop(). + +stream_loop() -> + stream_loop(orddict:new()). + +stream_loop(Acc) -> + receive {_Ref, done} -> + {ok, orddict:to_list(Acc)}; + {_Ref, {keys, Keys}} -> + Acc2 = orddict:update(keys, fun(Existing) -> Existing++Keys end, Keys, Acc), + stream_loop(Acc2); + {_Ref, {results, Results}} -> + Acc2 = orddict:update(results, fun(Existing) -> Existing++Results end, Results, Acc), + stream_loop(Acc2); + {_Ref, Res} when is_list(Res) -> + Keys = proplists:get_value(keys, Res, []), + Continuation = proplists:get_value(continuation, Res), + Acc2 = orddict:update(keys, fun(Existing) -> Existing++Keys end, [], Acc), + Acc3 = orddict:store(continuation, Continuation, Acc2), + stream_loop(Acc3); + {_Ref, Wat} -> + lager:info("got a wat ~p", [Wat]), + stream_loop(Acc) + end. + +pb_query(Pid, {Field, Val}, Opts) -> + riakc_pb_socket:get_index_eq(Pid, ?BUCKET, Field, Val, Opts); +pb_query(Pid, {Field, Start, End}, Opts) -> + riakc_pb_socket:get_index_range(Pid, ?BUCKET, Field, Start, End, Opts). + +http_stream(NodePath, Query, Opts) -> + http_query(NodePath, Query, Opts, stream). + +http_query(NodePath, Q) -> + http_query(NodePath, Q, []). + +http_query(NodePath, Query, Opts) -> + http_query(NodePath, Query, Opts, undefined). + +http_query(NodePath, {Field, Value}, Opts, Pid) -> + QString = opts_to_qstring(Opts, []), + Url = url("~s/buckets/~s/index/~s/~s~s", [NodePath, ?BUCKET, Field, Value, QString]), + http_get(Url, Pid); +http_query(NodePath, {Field, Start, End}, Opts, Pid) -> + QString = opts_to_qstring(Opts, []), + Url = url("~s/buckets/~s/index/~s/~s/~s~s", [NodePath, ?BUCKET, Field, Start, End, QString]), + http_get(Url, Pid). + +url(Format, Elements) -> + Path = io_lib:format(Format, Elements), + lists:flatten(Path). + +http_get(Url, undefined) -> + lager:info("getting ~p", [Url]), + {ok,{{"HTTP/1.1",200,"OK"}, _, Body}} = httpc:request(Url), + {struct, Result} = mochijson2:decode(Body), + Result; +http_get(Url, stream) -> + lager:info("streaming ~p", [Url]), + {ok, Ref} = httpc:request(get, {Url, []}, [], [{stream, self}, {sync, false}]), + http_stream_loop(Ref, []). + +opts_to_qstring([], QString) -> + QString; +opts_to_qstring([Opt|Rest], []) -> + QOpt = opt_to_string("?", Opt), + opts_to_qstring(Rest, QOpt); +opts_to_qstring([Opt|Rest], QString) -> + QOpt = opt_to_string("&", Opt), + opts_to_qstring(Rest, QString++QOpt). + +opt_to_string(Sep, {Name, Value}) when is_integer(Value) -> + io_lib:format(Sep++"~s=~p", [Name, Value]); +opt_to_string(Sep, {Name, Value})-> + io_lib:format(Sep++"~s=~s", [Name, Value]); +opt_to_string(Sep, Name) -> + io_lib:format(Sep++"~s=~s", [Name, true]). + +http_stream_loop(Ref, Acc) -> + receive {http, {Ref, stream_end, _Headers}} -> + {struct, Result} = mochijson2:decode(lists:flatten(lists:reverse(Acc))), + Result; + {http, {Ref, stream_start, _Headers}} -> + http_stream_loop(Ref, Acc); + {http, {Ref, stream, Body}} -> + http_stream_loop(Ref, [binary_to_list(Body) | Acc]) + end. diff --git a/tests/verify_2i_limit.erl b/tests/verify_2i_limit.erl new file mode 100644 index 00000000..cd5e4b28 --- /dev/null +++ b/tests/verify_2i_limit.erl @@ -0,0 +1,63 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(verify_2i_limit). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). +-import(secondary_index_tests, [put_an_object/2, int_to_key/1, + stream_pb/3, http_query/3]). +-define(BUCKET, <<"2ibucket">>). +-define(FOO, <<"foo">>). +-define(MAX_RESULTS, 50). + +confirm() -> + inets:start(), + + Nodes = rt:build_cluster(3), + ?assertEqual(ok, (rt:wait_until_nodes_ready(Nodes))), + + RiakHttp = rt:http_url(hd(Nodes)), + PBPid = rt:pbc(hd(Nodes)), + + [put_an_object(PBPid, N) || N <- lists:seq(0, 100)], + + ExpectedKeys = lists:sort([int_to_key(N) || N <- lists:seq(0, 100)]), + {FirstHalf, Rest} = lists:split(?MAX_RESULTS, ExpectedKeys), + Q = {<<"$key">>, int_to_key(0), int_to_key(999)}, + + %% PB + {ok, PBRes} = stream_pb(PBPid, Q, [{max_results, ?MAX_RESULTS}]), + ?assertEqual(FirstHalf, proplists:get_value(keys, PBRes, [])), + PBContinuation = proplists:get_value(continuation, PBRes), + + {ok, PBKeys2} = stream_pb(PBPid, Q, [{continuation, PBContinuation}]), + ?assertEqual(Rest, proplists:get_value(keys, PBKeys2, [])), + + %% HTTP + HttpRes = http_query(RiakHttp, Q, [{max_results, ?MAX_RESULTS}]), + ?assertEqual(FirstHalf, proplists:get_value(<<"keys">>, HttpRes, [])), + HttpContinuation = proplists:get_value(<<"continuation">>, HttpRes), + ?assertEqual(PBContinuation, HttpContinuation), + + HttpRes2 = http_query(RiakHttp, Q, [{continuation, HttpContinuation}]), + ?assertEqual(Rest, proplists:get_value(<<"keys">>, HttpRes2, [])), + + riakc_pb_socket:stop(PBPid), + pass. diff --git a/tests/verify_2i_returnterms.erl b/tests/verify_2i_returnterms.erl new file mode 100644 index 00000000..6e526a82 --- /dev/null +++ b/tests/verify_2i_returnterms.erl @@ -0,0 +1,78 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(verify_2i_returnterms). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). +-import(secondary_index_tests, [put_an_object/2, put_an_object/4, int_to_key/1, + stream_pb/3, http_query/3]). +-define(BUCKET, <<"2ibucket">>). +-define(FOO, <<"foo">>). +-define(Q_OPTS, [{return_terms, true}]). + +confirm() -> + inets:start(), + + Nodes = rt:build_cluster(3), + ?assertEqual(ok, (rt:wait_until_nodes_ready(Nodes))), + + RiakHttp = rt:http_url(hd(Nodes)), + PBPid = rt:pbc(hd(Nodes)), + + [put_an_object(PBPid, N) || N <- lists:seq(0, 100)], + [put_an_object(PBPid, int_to_key(N), N, ?FOO) || N <- lists:seq(101, 200)], + + %% Bucket, key, and index_eq queries should ignore `return_terms' + ExpectedKeys = lists:sort([int_to_key(N) || N <- lists:seq(0, 200)]), + assertEqual(RiakHttp, PBPid, ExpectedKeys, {<<"$key">>, int_to_key(0), int_to_key(999)}, ?Q_OPTS, keys), + assertEqual(RiakHttp, PBPid, ExpectedKeys, { <<"$bucket">>, ?BUCKET}, ?Q_OPTS, keys), + + ExpectedFooKeys = lists:sort([int_to_key(N) || N <- lists:seq(101, 200)]), + assertEqual(RiakHttp, PBPid, ExpectedFooKeys, {<<"field1_bin">>, ?FOO}, ?Q_OPTS, keys), + + %% Note: not sorted by key, but by value (the int index) + %% Should return "val, key" pairs + ExpectedRangeResults = [{list_to_binary(integer_to_list(N)), int_to_key(N)} || N <- lists:seq(1, 100)], + assertEqual(RiakHttp, PBPid, ExpectedRangeResults, {<<"field2_int">>, "1", "100"}, ?Q_OPTS, results), + + riakc_pb_socket:stop(PBPid), + pass. + +%% Check the PB result against our expectations +%% and the non-streamed HTTP +assertEqual(Http, PB, Expected, Query, Opts, ResultKey) -> + {ok, PBRes} = stream_pb(PB, Query, Opts), + PBKeys = proplists:get_value(ResultKey, PBRes, []), + HTTPRes = http_query(Http, Query, Opts), + HTTPResults0 = proplists:get_value(atom_to_binary(ResultKey, latin1), HTTPRes, []), + HTTPResults = decode_http_results(ResultKey, HTTPResults0), + ?assertEqual(Expected, PBKeys), + ?assertEqual(Expected, HTTPResults). + +decode_http_results(keys, Keys) -> + Keys; +decode_http_results(results, Results) -> + decode_http_results(Results, []); + +decode_http_results([], Acc) -> + lists:reverse(Acc); +decode_http_results([{struct, [Res]} | Rest ], Acc) -> + decode_http_results(Rest, [Res | Acc]). + diff --git a/tests/verify_2i_stream.erl b/tests/verify_2i_stream.erl new file mode 100644 index 00000000..d4974246 --- /dev/null +++ b/tests/verify_2i_stream.erl @@ -0,0 +1,68 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(verify_2i_stream). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). +-import(secondary_index_tests, [put_an_object/2, put_an_object/4, int_to_key/1, + stream_pb/2, http_query/2, http_stream/3]). +-define(BUCKET, <<"2ibucket">>). +-define(FOO, <<"foo">>). + +confirm() -> + inets:start(), + + Nodes = rt:build_cluster(3), + ?assertEqual(ok, (rt:wait_until_nodes_ready(Nodes))), + + RiakHttp = rt:http_url(hd(Nodes)), + PBPid = rt:pbc(hd(Nodes)), + + [put_an_object(PBPid, N) || N <- lists:seq(0, 100)], + [put_an_object(PBPid, int_to_key(N), N, ?FOO) || N <- lists:seq(101, 200)], + + ExpectedKeys = lists:sort([int_to_key(N) || N <- lists:seq(0, 200)]), + assertEqual(RiakHttp, PBPid, ExpectedKeys, {<<"$key">>, int_to_key(0), int_to_key(999)}), + assertEqual(RiakHttp, PBPid, ExpectedKeys, { <<"$bucket">>, ?BUCKET}), + + ExpectedFooKeys = lists:sort([int_to_key(N) || N <- lists:seq(101, 200)]), + assertEqual(RiakHttp, PBPid, ExpectedFooKeys, {<<"field1_bin">>, ?FOO}), + + %% Note: not sorted by key, but by value (the int index) + ExpectedRangeKeys = [int_to_key(N) || N <- lists:seq(1, 100)], + assertEqual(RiakHttp, PBPid, ExpectedRangeKeys, {<<"field2_int">>, "1", "100"}), + + riakc_pb_socket:stop(PBPid), + pass. + +%% Check the PB result against our expectations +%% and the non-streamed HTTP +assertEqual(Http, PB, Expected, Query) -> + {ok, PBRes} = stream_pb(PB, Query), + PBKeys = proplists:get_value(keys, PBRes, []), + HTTPRes = http_query(Http, Query), + StreamHTTPRes = http_stream(Http, Query, []), + lager:info("Got ~p.", [StreamHTTPRes]), + HTTPKeys = proplists:get_value(<<"keys">>, HTTPRes, []), + StreamHttpKeys = proplists:get_value(<<"keys">>, StreamHTTPRes, []), + ?assertEqual(Expected, PBKeys), + ?assertEqual(Expected, HTTPKeys), + ?assertEqual(Expected, StreamHttpKeys). + From e85afb3927f5ce7da52352b3d48f97767dd1ce3f Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Thu, 25 Apr 2013 09:06:13 +0100 Subject: [PATCH 2/6] Update v1 2i tests for new result format Remove dev / debug lager output --- tests/secondary_index_tests.erl | 4 +- tests/verify_2i_stream.erl | 1 - tests/verify_2i_v2query.erl | 163 ++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 tests/verify_2i_v2query.erl diff --git a/tests/secondary_index_tests.erl b/tests/secondary_index_tests.erl index f94335cc..e9e5cf4f 100644 --- a/tests/secondary_index_tests.erl +++ b/tests/secondary_index_tests.erl @@ -71,7 +71,7 @@ confirm() -> assertExactQuery(Pid, Expected, Index, Value) -> lager:info("Searching Index ~p for ~p", [Index, Value]), - {ok, Results} = riakc_pb_socket:get_index(Pid, ?BUCKET, Index, Value), + {ok, {keys, Results}} = riakc_pb_socket:get_index(Pid, ?BUCKET, Index, Value), ActualKeys = lists:sort(Results), lager:info("Expected: ~p", [Expected]), lager:info("Actual : ~p", [ActualKeys]), @@ -79,7 +79,7 @@ assertExactQuery(Pid, Expected, Index, Value) -> assertRangeQuery(Pid, Expected, Index, StartValue, EndValue) -> lager:info("Searching Index ~p for ~p-~p", [Index, StartValue, EndValue]), - {ok, Results} = riakc_pb_socket:get_index(Pid, ?BUCKET, Index, StartValue, EndValue), + {ok, {keys, Results}} = riakc_pb_socket:get_index(Pid, ?BUCKET, Index, StartValue, EndValue), ActualKeys = lists:sort(Results), lager:info("Expected: ~p", [Expected]), lager:info("Actual : ~p", [ActualKeys]), diff --git a/tests/verify_2i_stream.erl b/tests/verify_2i_stream.erl index d4974246..b0944059 100644 --- a/tests/verify_2i_stream.erl +++ b/tests/verify_2i_stream.erl @@ -59,7 +59,6 @@ assertEqual(Http, PB, Expected, Query) -> PBKeys = proplists:get_value(keys, PBRes, []), HTTPRes = http_query(Http, Query), StreamHTTPRes = http_stream(Http, Query, []), - lager:info("Got ~p.", [StreamHTTPRes]), HTTPKeys = proplists:get_value(<<"keys">>, HTTPRes, []), StreamHttpKeys = proplists:get_value(<<"keys">>, StreamHTTPRes, []), ?assertEqual(Expected, PBKeys), diff --git a/tests/verify_2i_v2query.erl b/tests/verify_2i_v2query.erl new file mode 100644 index 00000000..cb81af71 --- /dev/null +++ b/tests/verify_2i_v2query.erl @@ -0,0 +1,163 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(verify_2i_v2query). +-behavior(riak_test). + + +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). +-import(secondary_index_tests, [put_an_object/2, put_an_object/4, + int_to_key/1, int_to_field1_bin/1, + stream_pb/3, http_query/3, http_stream/3, + pb_query/3]). +-define(BUCKET, <<"2ibucket">>). +-define(FOO, <<"foo">>). + +confirm() -> + inets:start(), + ?assert(eqc:quickcheck(numtests(100, prop_query()))), + pass. + +prop_query() -> + ?FORALL({Query, Options, Protocol}, + {q(), options(), protocol()}, + begin + + Nodes = rt:build_cluster(3), + ?assertEqual(ok, (rt:wait_until_nodes_ready(Nodes))), + + RiakHttp = rt:http_url(hd(Nodes)), + PBPid = rt:pbc(hd(Nodes)), + + GetProtocol = fun(http) -> {http, RiakHttp}; + (pb) -> {pb, PBPid} end, + + [put_an_object(PBPid, N) || N <- lists:seq(0, 100)], + [put_an_object(PBPid, int_to_key(N), N, ?FOO) || N <- lists:seq(101, 200)], + + Expected = get_expected(Query, Options), + Result = execute(GetProtocol(Protocol), Query, Options), + AllResults = case more_results(Query, Options, Expected) of + true -> get_all_results(Query, Options, Result); + false -> Result + end, + ?WHENFAIL( + begin + io:format("Query ~p~n", [Query]), + io:format("Options ~p~n", [Options]), + io:format("Protocol ~p~n", [Protocol]) + end, + results_equal(Expected, AllResults)) + end). + +q() -> + oneof([range(), eq()]). + +range() -> + ?LET({Field, Start}, {range_field(), range_start()}, + {?BUCKET, Field, range_start(Field, Start), range_end(Field, Start)}). + +%% Less than the range size, please +range_start() -> + ?SUCHTHAT(N, nat(), N < 100). + +range_start(<<"$key">>, N) -> + int_to_key(N); +range_start(<<"field1_bin">>, N) -> + int_to_field1_bin(N); +range_start(<<"field2_int">>, N) -> + N. + +range_end(<<"$key">>, N) -> + ?LET(X, choose(N, 100), int_to_key(X)); +range_end(<<"field1_bin">>, N) -> + ?LET(X, choose(N, 100), int_to_field1_bin(X)); +range_end(<<"field2_int">>, N) -> + choose(N, 100). + +range_field() -> + oneof([<<"$key">>, <<"field1_bin">>]). + +eq() -> + ?LET(Field, eq_field(), {?BUCKET, Field, eq(Field)}). + +eq_field() -> + oneof([<<"$bucket">>, <<"$key">>, <<"field1_bin">>, <<"field2_int">>]). + +eq(<<"$bucket">>) -> + ?BUCKET; +eq(<<"$key">>) -> + ?LET(N, choose(1, 200), int_to_key(N)); +eq(<<"field1_bin">>) -> + ?LET(N, choose(1, 100), oneof([?FOO, int_to_field1_bin(N)])); +eq(<<"field2_int">>) -> + choose(1, 200). + +options() -> + ?LET({AllOptions, N}, {[stream, max_results(), return_terms()], choose(0, 3)}, + %% get N options from list + gen_options(AllOptions, N, [])). + +gen_options(_, 0, Opts) -> + Opts; +gen_options(L, N, Opts) -> + Elem = safe_rand(N), + L2 = lists:delete(Elem, L), + gen_options(L2, N-1, [Elem | Opts]). + +safe_rand(1) -> + 1; +safe_rand(E) -> + crypto:rand_uniform(1, E). + +max_results() -> + choose(1, 100). + +return_terms() -> + bool(). + +protocol() -> + oneof([pb, http]). + +results_equal(Expected, AllResults) -> + %% TODO + Expected == AllResults. + +get_expected(Query, Options) -> + %% TODO + [Query, Options]. + +execute({http, Http}, Query, Options) -> + http_query(Http, Query, Options); +execute({pb, Pid}, Query, Options) -> + pb_query(Pid, Query, Options). + +more_results(_Query, _Options, _Expected) -> + false. + +get_all_results(_Query, _Options, _Result) -> + []. + + + + From 0da138f70760357a0d209322efd2fcada5d32476 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Thu, 25 Apr 2013 13:58:14 +0100 Subject: [PATCH 3/6] Remove unfinished / unworking eqc test --- tests/verify_2i_v2query.erl | 163 ------------------------------------ 1 file changed, 163 deletions(-) delete mode 100644 tests/verify_2i_v2query.erl diff --git a/tests/verify_2i_v2query.erl b/tests/verify_2i_v2query.erl deleted file mode 100644 index cb81af71..00000000 --- a/tests/verify_2i_v2query.erl +++ /dev/null @@ -1,163 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2013 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- --module(verify_2i_v2query). --behavior(riak_test). - - --include_lib("eqc/include/eqc.hrl"). --include_lib("eunit/include/eunit.hrl"). - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --import(secondary_index_tests, [put_an_object/2, put_an_object/4, - int_to_key/1, int_to_field1_bin/1, - stream_pb/3, http_query/3, http_stream/3, - pb_query/3]). --define(BUCKET, <<"2ibucket">>). --define(FOO, <<"foo">>). - -confirm() -> - inets:start(), - ?assert(eqc:quickcheck(numtests(100, prop_query()))), - pass. - -prop_query() -> - ?FORALL({Query, Options, Protocol}, - {q(), options(), protocol()}, - begin - - Nodes = rt:build_cluster(3), - ?assertEqual(ok, (rt:wait_until_nodes_ready(Nodes))), - - RiakHttp = rt:http_url(hd(Nodes)), - PBPid = rt:pbc(hd(Nodes)), - - GetProtocol = fun(http) -> {http, RiakHttp}; - (pb) -> {pb, PBPid} end, - - [put_an_object(PBPid, N) || N <- lists:seq(0, 100)], - [put_an_object(PBPid, int_to_key(N), N, ?FOO) || N <- lists:seq(101, 200)], - - Expected = get_expected(Query, Options), - Result = execute(GetProtocol(Protocol), Query, Options), - AllResults = case more_results(Query, Options, Expected) of - true -> get_all_results(Query, Options, Result); - false -> Result - end, - ?WHENFAIL( - begin - io:format("Query ~p~n", [Query]), - io:format("Options ~p~n", [Options]), - io:format("Protocol ~p~n", [Protocol]) - end, - results_equal(Expected, AllResults)) - end). - -q() -> - oneof([range(), eq()]). - -range() -> - ?LET({Field, Start}, {range_field(), range_start()}, - {?BUCKET, Field, range_start(Field, Start), range_end(Field, Start)}). - -%% Less than the range size, please -range_start() -> - ?SUCHTHAT(N, nat(), N < 100). - -range_start(<<"$key">>, N) -> - int_to_key(N); -range_start(<<"field1_bin">>, N) -> - int_to_field1_bin(N); -range_start(<<"field2_int">>, N) -> - N. - -range_end(<<"$key">>, N) -> - ?LET(X, choose(N, 100), int_to_key(X)); -range_end(<<"field1_bin">>, N) -> - ?LET(X, choose(N, 100), int_to_field1_bin(X)); -range_end(<<"field2_int">>, N) -> - choose(N, 100). - -range_field() -> - oneof([<<"$key">>, <<"field1_bin">>]). - -eq() -> - ?LET(Field, eq_field(), {?BUCKET, Field, eq(Field)}). - -eq_field() -> - oneof([<<"$bucket">>, <<"$key">>, <<"field1_bin">>, <<"field2_int">>]). - -eq(<<"$bucket">>) -> - ?BUCKET; -eq(<<"$key">>) -> - ?LET(N, choose(1, 200), int_to_key(N)); -eq(<<"field1_bin">>) -> - ?LET(N, choose(1, 100), oneof([?FOO, int_to_field1_bin(N)])); -eq(<<"field2_int">>) -> - choose(1, 200). - -options() -> - ?LET({AllOptions, N}, {[stream, max_results(), return_terms()], choose(0, 3)}, - %% get N options from list - gen_options(AllOptions, N, [])). - -gen_options(_, 0, Opts) -> - Opts; -gen_options(L, N, Opts) -> - Elem = safe_rand(N), - L2 = lists:delete(Elem, L), - gen_options(L2, N-1, [Elem | Opts]). - -safe_rand(1) -> - 1; -safe_rand(E) -> - crypto:rand_uniform(1, E). - -max_results() -> - choose(1, 100). - -return_terms() -> - bool(). - -protocol() -> - oneof([pb, http]). - -results_equal(Expected, AllResults) -> - %% TODO - Expected == AllResults. - -get_expected(Query, Options) -> - %% TODO - [Query, Options]. - -execute({http, Http}, Query, Options) -> - http_query(Http, Query, Options); -execute({pb, Pid}, Query, Options) -> - pb_query(Pid, Query, Options). - -more_results(_Query, _Options, _Expected) -> - false. - -get_all_results(_Query, _Options, _Result) -> - []. - - - - From 541158a9652d47fcdaffbe59eed0a5715c737971 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Fri, 3 May 2013 18:38:24 +0100 Subject: [PATCH 4/6] Test cs bucket query feature --- tests/verify_cs_bucket.erl | 87 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/verify_cs_bucket.erl diff --git a/tests/verify_cs_bucket.erl b/tests/verify_cs_bucket.erl new file mode 100644 index 00000000..189a525c --- /dev/null +++ b/tests/verify_cs_bucket.erl @@ -0,0 +1,87 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @ doc tests the new CS Bucket fold message + +-module(verify_cs_bucket). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). +-import(secondary_index_tests, [put_an_object/2, int_to_key/1]). +-define(BUCKET, <<"2ibucket">>). +-define(FOO, <<"foo">>). + +confirm() -> + Nodes = rt:build_cluster(3), + ?assertEqual(ok, (rt:wait_until_nodes_ready(Nodes))), + + PBPid = rt:pbc(hd(Nodes)), + + [put_an_object(PBPid, N) || N <- lists:seq(0, 200)], + + ExpectedKeys = lists:sort([int_to_key(N) || N <- lists:seq(0, 200)]), + + undefined = assertEqual(PBPid, ExpectedKeys, ?BUCKET, [{start_key, int_to_key(0)}]), + undefined = assertEqual(PBPid, tl(ExpectedKeys), ?BUCKET, [{start_key, int_to_key(0)}, {start_incl, false}]), + undefined = assertEqual(PBPid, [int_to_key(104)], ?BUCKET, [{start_key, int_to_key(103)}, + {end_key, int_to_key(105)}, + {start_incl, false}, + {end_incl, false}]), + + %% Limit / continuations + + Continuation1 = assertEqual(PBPid, lists:sublist(ExpectedKeys, 20), ?BUCKET, [{start_key, int_to_key(0)}, {max_results, 20}]), + Continuation2 = assertEqual(PBPid, lists:sublist(ExpectedKeys, 21, 20), ?BUCKET, + [{start_key, int_to_key(0)}, {max_results, 20}, {continuation, Continuation1}]), + undefined = assertEqual(PBPid, lists:sublist(ExpectedKeys, 41, 200), ?BUCKET, [{continuation, Continuation2}, {max_results, 200}]), + + riakc_pb_socket:stop(PBPid), + pass. + +%% Check the PB result against our expectations +%% and the non-streamed HTTP +assertEqual(PB, Expected, Bucket, Opts) -> + {ok, PBRes} = stream_pb(PB, Bucket, Opts), + PBObjects = proplists:get_value(objects, PBRes, []), + Keys = [riakc_obj:key(Obj) || Obj <- PBObjects], + ?assertEqual(Expected, Keys), + proplists:get_value(continuation, PBRes). + + +stream_pb(Pid, Bucket, Opts) -> + riakc_pb_socket:cs_bucket_fold(Pid, Bucket, Opts), + stream_loop(). + +stream_loop() -> + stream_loop(orddict:new()). + +stream_loop(Acc) -> + receive {_Ref, done} -> + {ok, orddict:to_list(Acc)}; + {_Ref, {objects, Objects}} -> + Acc2 = orddict:update(objects, fun(Existing) -> Existing++Objects end, Objects, Acc), + stream_loop(Acc2); + {_Ref, Res} when is_list(Res) -> + Objects = proplists:get_value(objects, Res, []), + Continuation = proplists:get_value(continuation, Res), + Acc2 = orddict:update(objects, fun(Existing) -> Existing++Objects end, Objects, Acc), + Acc3 = orddict:store(continuation, Continuation, Acc2), + stream_loop(Acc3) + end. From 98d68580f35ae5217d3342fb6f8159f6a9a578c8 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Thu, 23 May 2013 10:35:56 +0100 Subject: [PATCH 5/6] Add test for dropped duplicate key after continuation bug --- tests/verify_2i_limit.erl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/verify_2i_limit.erl b/tests/verify_2i_limit.erl index cd5e4b28..039f6946 100644 --- a/tests/verify_2i_limit.erl +++ b/tests/verify_2i_limit.erl @@ -22,7 +22,7 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). -import(secondary_index_tests, [put_an_object/2, int_to_key/1, - stream_pb/3, http_query/3]). + stream_pb/3, http_query/3, pb_query/3]). -define(BUCKET, <<"2ibucket">>). -define(FOO, <<"foo">>). -define(MAX_RESULTS, 50). @@ -59,5 +59,23 @@ confirm() -> HttpRes2 = http_query(RiakHttp, Q, [{continuation, HttpContinuation}]), ?assertEqual(Rest, proplists:get_value(<<"keys">>, HttpRes2, [])), + %% Multiple indexes for single key + O1 = riakc_obj:new(?BUCKET, <<"bob">>, <<"1">>), + Md = riakc_obj:get_metadata(O1), + Md2 = riakc_obj:set_secondary_index(Md, {{integer_index, "i1"}, [300, 301, 302]}), + O2 = riakc_obj:update_metadata(O1, Md2), + riakc_pb_socket:put(PBPid, O2), + + MQ = {"i1_int", 300, 302}, + {ok, Res} = pb_query(PBPid, MQ, [{max_results, 2}, return_terms]), + + ?assertEqual([{<<"300">>, <<"bob">>}, + {<<"301">>, <<"bob">>}], proplists:get_value(results, Res)), + + {ok, Res2} = pb_query(PBPid, MQ, [{max_results, 2}, return_terms, + {continuation, proplists:get_value(continuation, Res)}]), + + ?assertEqual({results,[{<<"302">>,<<"bob">>}]}, Res2), + riakc_pb_socket:stop(PBPid), pass. From cb75af39ab7115b64abf8c4274fb802a99dd12b9 Mon Sep 17 00:00:00 2001 From: "Engel A. Sanchez" Date: Wed, 5 Jun 2013 17:24:59 -0400 Subject: [PATCH 6/6] Add wait until really deleted function I was having test failures due to the delete operation timing out silently. This was due to dependencies not update by rebar in the end. But in any case, this new function is more robust in detecting when tombstones are really reaped instead of a sleep call. Also, matching delete operation so it's noisy when it fails. --- src/rt.erl | 36 +++++++++++++++++++++++++++++++++ tests/secondary_index_tests.erl | 10 ++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/rt.erl b/src/rt.erl index 0956b35d..aebad92c 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -84,6 +84,7 @@ pbc_write/4, pbc_put_dir/3, pbc_put_file/4, + pbc_wait_until_really_deleted/5, pmap/2, remove/2, riak/2, @@ -966,6 +967,41 @@ pbc_put_dir(Pid, Bucket, Dir) -> [pbc_put_file(Pid, Bucket, list_to_binary(F), filename:join([Dir, F])) || F <- Files]. +%% @doc Wait until the given keys have been really, really deleted. +%% Useful when you care about the keys not being there. Delete simply writes +%% tombstones under the given keys, so those are still seen by key folding +%% operations. +pbc_wait_until_really_deleted(_Pid, _Bucket, [], _Delay, _Retries) -> + ok; +pbc_wait_until_really_deleted(_Pid, _Bucket, _Keys, _Delay, Retries) when Retries < 1 -> + {error, timeout}; +pbc_wait_until_really_deleted(Pid, Bucket, Keys, Delay, Retries) + when is_integer(Retries), Retries > 0, is_list(Keys) -> + StillThere = + fun(K) -> + Res = riakc_pb_socket:get(Pid, Bucket, K, + [{r, 1}, + {notfound_ok, false}, + {basic_quorum, false}, + deletedvclock]), + case Res of + {error, notfound} -> + false; + _ -> + %% Tombstone still around + true + end + end, + case lists:filter(StillThere, Keys) of + [] -> + ok; + NewKeys -> + timer:sleep(Delay), + pbc_wait_until_really_deleted(Pid, Bucket, NewKeys, Delay, Retries-1) + end. + + + %% @doc Returns HTTP URL information for a list of Nodes http_url(Nodes) when is_list(Nodes) -> [begin diff --git a/tests/secondary_index_tests.erl b/tests/secondary_index_tests.erl index e9e5cf4f..9a91f5f8 100644 --- a/tests/secondary_index_tests.erl +++ b/tests/secondary_index_tests.erl @@ -42,11 +42,10 @@ confirm() -> assertRangeQuery(Pid, [<<"obj10">>, <<"obj11">>, <<"obj12">>], <<"$key">>, <<"obj10">>, <<"obj12">>), lager:info("Delete an object, verify deletion..."), - riakc_pb_socket:delete(Pid, ?BUCKET, <<"obj5">>), - riakc_pb_socket:delete(Pid, ?BUCKET, <<"obj11">>), - - lager:info("Sleeping for 5 seconds. Make sure the tombstone is reaped..."), - timer:sleep(5000), + ToDel = [<<"obj5">>, <<"obj11">>], + [?assertMatch(ok, riakc_pb_socket:delete(Pid, ?BUCKET, K)) || K <- ToDel], + lager:info("Make sure the tombstone is reaped..."), + ?assertMatch(ok, rt:pbc_wait_until_really_deleted(Pid, ?BUCKET, ToDel, 1000, 20)), assertExactQuery(Pid, [], <<"field1_bin">>, <<"val5">>), assertExactQuery(Pid, [], <<"field2_int">>, <<"5">>), @@ -68,7 +67,6 @@ confirm() -> TestIdxVal), pass. - assertExactQuery(Pid, Expected, Index, Value) -> lager:info("Searching Index ~p for ~p", [Index, Value]), {ok, {keys, Results}} = riakc_pb_socket:get_index(Pid, ?BUCKET, Index, Value),