valitydev/riak_test
14
0
Fork 0
mirror of https://github.com/valitydev/riak_test.git synced 2024-11-06 08:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
0c1e5cf1f8
riak_test/tests/yz_schema_change_reset.erl
Zeeshan Lakhani d269a2fd98 - search expect refactor and associated cleanup 
- handle wait_for_index properly after index creation and cleanup write_data
  to include commit
2016-05-25 08:59:32 -04:00

306 lines
13 KiB
Erlang
Raw Blame History

%% -------------------------------------------------------------------
%%
%% Copyright (c) 2015 Basho Technologies, Inc.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%%--------------------------------------------------------------------
-module(yz_schema_change_reset).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("riakc/include/riakc.hrl").
-define(GET(K,L), proplists:get_value(K, L)).
-define(INDEX, <<"test_schema_change_reset">>).
-define(TYPE, <<"test_schema_change">>).
-define(BUCKET1, <<"test_schema_change_reset">>).
-define(BUCKET2, {?TYPE, <<"test_schema_change_reset_2">>}).
-define(SCHEMANAME, <<"test">>).
-define(TEST_SCHEMA,
<<"<schema name=\"test\" version=\"1.5\">
<fields>
<dynamicField name=\"*_foo_register\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<dynamicField name=\"*\" type=\"ignored\"/>
<field name=\"_yz_id\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" required=\"true\" multiValued=\"false\"/>
<field name=\"_yz_ed\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_pn\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_fpn\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_vtag\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_rt\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"_yz_rk\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"_yz_rb\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"_yz_err\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"text\" type=\"text_general\" indexed=\"true\" stored=\"false\" multiValued=\"true\"/>
<field name=\"age\" type=\"int\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
</fields>
<uniqueKey>_yz_id</uniqueKey>
<types>
<fieldType name=\"ignored\" indexed=\"false\" stored=\"false\" multiValued=\"false\" class=\"solr.StrField\" />
<fieldType name=\"_yz_str\" class=\"solr.StrField\" sortMissingLast=\"true\" />
<fieldType name=\"string\" class=\"solr.StrField\" sortMissingLast=\"true\" />
<fieldType name=\"int\" class=\"solr.TrieIntField\" precisionStep=\"0\" positionIncrementGap=\"0\" />
<fieldType name=\"text_general\" class=\"solr.TextField\" positionIncrementGap=\"100\">
<analyzer type=\"index\">
<tokenizer class=\"solr.StandardTokenizerFactory\"/>
<filter class=\"solr.StopFilterFactory\" ignoreCase=\"true\" words=\"stopwords.txt\" enablePositionIncrements=\"true\" />
<filter class=\"solr.LowerCaseFilterFactory\"/>
</analyzer>
<analyzer type=\"query\">
<tokenizer class=\"solr.StandardTokenizerFactory\"/>
<filter class=\"solr.StopFilterFactory\" ignoreCase=\"true\" words=\"stopwords.txt\" enablePositionIncrements=\"true\" />
<filter class=\"solr.SynonymFilterFactory\" synonyms=\"synonyms.txt\" ignoreCase=\"true\" expand=\"true\"/>
<filter class=\"solr.LowerCaseFilterFactory\"/>
</analyzer>
</fieldType>
</types>
</schema>">>).
-define(TEST_SCHEMA_UPDATE,
<<"<schema name=\"test\" version=\"1.5\">
<fields>
<dynamicField name=\"*_foo_register\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<dynamicField name=\"*_baz_counter\" type=\"int\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<dynamicField name=\"paths.*\" type=\"int\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<dynamicField name=\"*\" type=\"ignored\"/>
<field name=\"_yz_id\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" required=\"true\" multiValued=\"false\"/>
<field name=\"_yz_ed\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_pn\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_fpn\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_vtag\" type=\"_yz_str\" indexed=\"true\" multiValued=\"false\"/>
<field name=\"_yz_rt\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"_yz_rk\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"_yz_rb\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"_yz_err\" type=\"_yz_str\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"text\" type=\"text_general\" indexed=\"true\" stored=\"false\" multiValued=\"true\"/>
<field name=\"age\" type=\"string\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
<field name=\"hello_i\" type=\"int\" indexed=\"true\" stored=\"true\" multiValued=\"false\"/>
</fields>
<uniqueKey>_yz_id</uniqueKey>
<types>
<fieldType name=\"ignored\" indexed=\"false\" stored=\"false\" multiValued=\"false\" class=\"solr.StrField\" />
<fieldType name=\"_yz_str\" class=\"solr.StrField\" sortMissingLast=\"true\" />
<fieldType name=\"string\" class=\"solr.StrField\" sortMissingLast=\"true\" />
<fieldType name=\"int\" class=\"solr.TrieIntField\" precisionStep=\"0\" positionIncrementGap=\"0\" />
<fieldType name=\"text_general\" class=\"solr.TextField\" positionIncrementGap=\"100\">
<analyzer type=\"index\">
<tokenizer class=\"solr.StandardTokenizerFactory\"/>
<filter class=\"solr.StopFilterFactory\" ignoreCase=\"true\" words=\"stopwords.txt\" enablePositionIncrements=\"true\" />
<filter class=\"solr.LowerCaseFilterFactory\"/>
</analyzer>
<analyzer type=\"query\">
<tokenizer class=\"solr.StandardTokenizerFactory\"/>
<filter class=\"solr.StopFilterFactory\" ignoreCase=\"true\" words=\"stopwords.txt\" enablePositionIncrements=\"true\" />
<filter class=\"solr.SynonymFilterFactory\" synonyms=\"synonyms.txt\" ignoreCase=\"true\" expand=\"true\"/>
<filter class=\"solr.LowerCaseFilterFactory\"/>
</analyzer>
</fieldType>
</types>
</schema>">>).
-define(SEQMAX, 20).
-define(CFG,
[{riak_core,
[
{ring_creation_size, 16}
]},
{riak_kv, [
{anti_entropy_concurrency, 8},
{anti_entropy_build_limit, {100, 1000}}
]},
{yokozuna,
[
{anti_entropy_tick, 1000},
{enabled, true}
]}
]).
confirm() ->
[Node1|_RestNodes] = Cluster = rt:build_cluster(4, ?CFG),
rt:wait_for_cluster_service(Cluster, yokozuna),
GenKeys = yokozuna_rt:gen_keys(?SEQMAX),
KeyCount = length(GenKeys),
lager:info("KeyCount ~p", [KeyCount]),
Pid = rt:pbc(rt:select_random(Cluster)),
lager:info("Write initial data to index ~p with schema ~p",
[?INDEX, ?SCHEMANAME]),
yokozuna_rt:write_data(Cluster, Pid, ?INDEX,
{?SCHEMANAME, ?TEST_SCHEMA},
?BUCKET1, GenKeys),
lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s",
[?TYPE, ?INDEX]),
rt:create_and_activate_bucket_type(Node1, ?TYPE, [{datatype, map},
{search_index, ?INDEX}]),
lager:info("Write and check age at integer per original schema"),
NewObj1A = riakc_obj:new(?BUCKET1, <<"keyA">>,
<<"{\"age\":26}">>,
"application/json"),
NewObj1B = riakc_obj:new(?BUCKET1, <<"keyB">>,
<<"{\"age\":99}">>,
"application/json"),
{ok, _ObjA} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]),
yokozuna_rt:commit(Cluster, ?INDEX),
{ok, _ObjB} = riakc_pb_socket:put(Pid, NewObj1B, [return_head]),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 2),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"age:26">>, {<<"age">>, <<"26">>}, []),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"age:99">>, {<<"age">>, <<"99">>}, []),
Map1 = riakc_map:update(
{<<"0_foo">>, register},
fun(R) ->
riakc_register:set(<<"44ab">>, R)
end, riakc_map:new()),
ok = riakc_pb_socket:update_type(
Pid,
?BUCKET2,
<<"keyMap1">>,
riakc_map:to_op(Map1)),
{ok, Map2} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>),
Map3 = riakc_map:update(
{<<"1_baz">>, counter},
fun(R) ->
riakc_counter:increment(10, R)
end, Map2),
ok = riakc_pb_socket:update_type(
Pid,
?BUCKET2,
<<"keyMap1">>,
riakc_map:to_op(Map3)),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"0_foo_register:44ab">>,
{<<"0_foo_register">>, <<"44ab">>},
[]),
lager:info("Expire and re-check count before updating schema"),
yokozuna_rt:expire_trees(Cluster),
yokozuna_rt:wait_for_aae(Cluster),
yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 3),
yokozuna_rt:override_schema(Pid, Cluster, ?INDEX, ?SCHEMANAME, ?TEST_SCHEMA_UPDATE),
lager:info("Write and check hello_i at integer per schema update"),
NewObj2 = riakc_obj:new(?BUCKET1, <<"key2">>,
<<"{\"hello_i\":36}">>,
"application/json"),
{ok, _Obj2} = riakc_pb_socket:put(Pid, NewObj2, [return_head]),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []),
lager:info("Write and check age at string per schema update"),
NewObj3 = riakc_obj:new(?BUCKET1, <<"key3">>,
<<"{\"age\":\"3jlkjkl\"}">>,
"application/json"),
{ok, _Obj3} = riakc_pb_socket:put(Pid, NewObj3, [return_head]),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"age:3jlkjkl">>, {<<"age">>, <<"3jlkjkl">>},
[]),
lager:info("Expire and re-check count to make sure we're correctly indexed
by the new schema"),
yokozuna_rt:expire_trees(Cluster),
yokozuna_rt:wait_for_aae(Cluster),
yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5),
ANode = rt:select_random(Cluster),
yokozuna_rt:search_expect(ANode, ?INDEX, <<"age">>, <<"*">>, 3),
lager:info("Re-Put because AAE won't find a diff even though the types
have changed, as it only compares based on bkey currently.
Also, this re-put will work as we have a default bucket (type)
with allow_mult=false... no siblings"),
{ok, _Obj4} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"age:26">>, {<<"age">>, <<"26">>}, []),
lager:info("Re-Put Map data by dec/inc counter to account for *change* and
allow previously unindexed counter to be searchable"),
{ok, Map4} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>),
Map5 = riakc_map:update(
{<<"1_baz">>, counter},
fun(R) ->
riakc_counter:decrement(0, R),
riakc_counter:increment(0, R)
end, Map4),
ok = riakc_pb_socket:update_type(
Pid,
?BUCKET2,
<<"keyMap1">>,
riakc_map:to_op(Map5)),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"0_foo_register:44ab">>,
{<<"0_foo_register">>, <<"44ab">>},
[]),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"1_baz_counter:10">>,
{<<"1_baz_counter">>, <<"10">>},
[]),
lager:info("Test nested json searches w/ unsearched fields ignored"),
NewObj5 = riakc_obj:new(?BUCKET1, <<"key4">>,
<<"{\"quip\":\"blashj3\",
\"paths\":{\"quip\":\"88\"}}">>,
"application/json"),
{ok, _Obj5} = riakc_pb_socket:put(Pid, NewObj5, [return_head]),
yokozuna_rt:commit(Cluster, ?INDEX),
yokozuna_rt:assert_search(Pid, Cluster, ?INDEX,
<<"paths.quip:88">>,
{<<"paths.quip">>, <<"88">>},
[]),
riakc_pb_socket:stop(Pid),
pass.
Reference in New Issue View Git Blame Copy Permalink