valitydev/riak_test
14
0
Fork 0
mirror of https://github.com/valitydev/riak_test.git synced 2024-11-06 08:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add a more sophisticated equality check to verify

Browse Source 
The partition repair test deletes all the data at a partition, and then
repairs it from neighbouring partitions. The subset of repaired data
that was originally coordinated by the deleted partition's vnode showed
up as `notfound` since the latest kv679 changes here
https://github.com/basho/riak_kv/pull/1643/. The reason is that the fix
in the KV repo adds a new actor to the repaired key's vclock. Prior to
this change `verify` in partition_repair.erl did a simple equality check
on the binary encoded riak_object values. This change takes into account
that a new actor may be in the vclock at the repaired vnode, and uses a
semantic equality check based on riak_object merge and riak object
equal.
This commit is contained in:
Russell Brown 2017-04-03 16:48:46 +01:00
parent 5db2943270
commit 853ae5fab2
1 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
tests/partition_repair.erl
View File

@ -151,7 +151,8 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) ->
lager:info("Verify ~p on ~p is fully repaired", [Partition, Node]),
Data2 = get_data({Partition, Node}),
{Verified, NotFound} = dict:fold(verify(Service, Data2), {0, []}, Stash),
{Verified, NotFound} = dict:fold(verify(Node, Service, Data2), {0, []}, Stash),
case NotFound of
[] -> ok;
_ ->
@ -170,7 +171,8 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) ->
{ok, [StashB]} = file:consult(StashPathB),
ExpectToVerifyB = dict:size(StashB),
BeforeData = get_data(B),
{VerifiedB, NotFoundB} = dict:fold(verify(Service, BeforeData), {0, []}, StashB),
{VerifiedB, NotFoundB} = dict:fold(verify(Node, Service, BeforeData), {0, []}, StashB),
case NotFoundB of
[] -> ok;
_ ->
@ -184,8 +186,10 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) ->
StashPathA = stash_path(Service, AfterP),
{ok, [StashA]} = file:consult(StashPathA),
ExpectToVerifyA = dict:size(StashA),
AfterData = get_data(A),
{VerifiedA, NotFoundA} = dict:fold(verify(Service, AfterData), {0, []}, StashA),
{VerifiedA, NotFoundA} = dict:fold(verify(Node, Service, AfterData), {0, []}, StashA),
case NotFoundA of
[] -> ok;
_ ->
@ -196,18 +200,38 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) ->
?assertEqual(ExpectToVerifyA, VerifiedA).
verify(riak_kv, DataAfterRepair) ->
verify(Node, riak_kv, DataAfterRepair) ->
fun(BKey, StashedValue, {Verified, NotFound}) ->
StashedData={BKey, StashedValue},
case dict:find(BKey, DataAfterRepair) of
error -> {Verified, [StashedData|NotFound]};
{ok, Value} ->
if Value == StashedValue -> {Verified+1, NotFound};
true -> {Verified, [StashedData|NotFound]}
%% NOTE: since kv679 fixes, the binary values may
%% not be equal where a new epoch-actor-entry has
%% been added to the repaired value in the vnode
case gte(Node, Value, StashedValue, BKey) of
true -> {Verified+1, NotFound};
false -> {Verified, [StashedData|NotFound]}
end
end
end.
%% @private gte checks that `Value' is _at least_ `StashedValue'. With
%% the changes for kv679 when a vnode receives a write of a key that
%% contains the vnode's id as an entry in the version vector, it adds
%% a new actor-epoch-entry to the version vector to guard against data
%% loss from repeated events (remember a VV history is supposed to be
%% unique!) This function then must deserialise the stashed and vnode
%% data and check that they are equal, or if not, the only difference
%% is an extra epoch actor in the vnode value's vclock.
gte(Node, Value, StashedData, {B, K}) ->
VnodeObject = riak_object:from_binary(B, K, Value),
StashedObject = riak_object:from_binary(B, K, StashedData),
%% NOTE: we need a ring and all that jazz for bucket props, needed
%% by merge, so use an RPC to merge on a riak node.
Merged = rpc:call(Node, riak_object, syntactic_merge, [VnodeObject, StashedObject]),
riak_object:equal(VnodeObject, Merged).
is_true(X) ->
X == true.
@ -269,4 +293,4 @@ backend_mod_dir(bitcask) ->
backend_mod_dir(eleveldb) ->
{riak_kv_eleveldb_backend, "leveldb"};
backend_mod_dir(leveled) ->
{riak_kv_leveled_backed, "leveled"}.
{riak_kv_leveled_backed, "leveled"}.