From f7f7e3fc7af2acd7aa0b033366e3f1261b4c5f90 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Mon, 4 Apr 2016 00:47:45 -0400 Subject: [PATCH 1/2] add Hyperloglog DT support and few minor test fixins Still WIP - Figure out correct riak_pb tag - Figure how best to support these changes! - expect buildbot tests to fail b/c live_node tests will need the new HLL shit runnin'! --- rebar.config | 2 +- src/riakc_counter.erl | 2 +- src/riakc_datatype.erl | 12 +++-- src/riakc_flag.erl | 2 +- src/riakc_hll.erl | 113 ++++++++++++++++++++++++++++++++++++++++ src/riakc_map.erl | 4 +- src/riakc_pb_socket.erl | 25 +++++++++ src/riakc_register.erl | 2 +- src/riakc_set.erl | 8 ++- 9 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 src/riakc_hll.erl diff --git a/rebar.config b/rebar.config index 916c03f3..76f7497f 100644 --- a/rebar.config +++ b/rebar.config @@ -13,7 +13,7 @@ ]}. {deps, [ - {riak_pb, "2.1.*", {git, "https://github.com/basho/riak_pb", {tag, "2.1.4.1"}}} + {riak_pb, ".*", {git, "git://github.com/basho/riak_pb", {branch, "feature-zl-hll_datatypes"}}} ]}. {edoc_opts, [ diff --git a/src/riakc_counter.erl b/src/riakc_counter.erl index bb29e609..023e36ba 100644 --- a/src/riakc_counter.erl +++ b/src/riakc_counter.erl @@ -31,7 +31,7 @@ -ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). --compile(export_all). +-export([gen_type/0, gen_op/0]). -endif. %% Callbacks diff --git a/src/riakc_datatype.erl b/src/riakc_datatype.erl index 6561d76d..dd6c5a10 100644 --- a/src/riakc_datatype.erl +++ b/src/riakc_datatype.erl @@ -2,7 +2,7 @@ %% %% riakc_datatype: Behaviour for eventually-consistent data-types %% -%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -30,11 +30,14 @@ -include_lib("eqc/include/eqc.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(QC_OUT(P), eqc:on_output(fun(Fmt, Args) -> io:format(user, Fmt, Args) end, P)). +-callback gen_type() -> eqc_gen:gen(datatype()). +-callback gen_op() -> eqc_gen:gen({atom(), [term()]}). -compile(export_all). -endif. --define(MODULES, [riakc_set, riakc_counter, riakc_flag, riakc_register, riakc_map]). +-define(MODULES, [riakc_set, riakc_counter, riakc_flag, riakc_register, + riakc_map, riakc_hll]). -export([module_for_type/1, module_for_term/1]). @@ -82,6 +85,7 @@ %% type. -spec module_for_type(Type::atom()) -> module(). module_for_type(set) -> riakc_set; +module_for_type(hll) -> riakc_hll; module_for_type(counter) -> riakc_counter; module_for_type(flag) -> riakc_flag; module_for_type(register) -> riakc_register; @@ -109,9 +113,9 @@ module_for_term(T) -> -define(F(Fmt, Args), lists:flatten(io_lib:format(Fmt, Args))). datatypes_test_() -> [{" prop_module_type() ", - ?_assertEqual(true, quickcheck(?QC_OUT(prop_module_type())))}] ++ + ?_assert(eqc:quickcheck(?QC_OUT(prop_module_type())))}] ++ [ {?F(" ~s(~s) ", [Prop, Mod]), - ?_assertEqual(true, quickcheck(?QC_OUT(eqc:testing_time(2, ?MODULE:Prop(Mod)))))} || + ?_assert(eqc:quickcheck(?QC_OUT(eqc:testing_time(2, ?MODULE:Prop(Mod)))))} || Prop <- ?MODPROPS, Mod <- ?MODULES ]. diff --git a/src/riakc_flag.erl b/src/riakc_flag.erl index 973e05cb..51b0eb85 100644 --- a/src/riakc_flag.erl +++ b/src/riakc_flag.erl @@ -33,7 +33,7 @@ -ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). --compile(export_all). +-export([gen_type/0, gen_op/0]). -endif. %% Callbacks diff --git a/src/riakc_hll.erl b/src/riakc_hll.erl new file mode 100644 index 00000000..843920cc --- /dev/null +++ b/src/riakc_hll.erl @@ -0,0 +1,113 @@ +%% ------------------------------------------------------------------- +%% +%% riakc_set: Eventually-consistent hyperloglog(set) type +%% +%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + + +%%@doc +-module(riakc_hll). +-behaviour(riakc_datatype). + +-ifdef(EQC). +-include_lib("eqc/include/eqc.hrl"). +-export([gen_type/0, gen_op/0]). +-endif. + +%% Callbacks +-export([new/0, new/1, new/2, + value/1, + to_op/1, + is_type/1, + type/0]). + +%% Operations +-export([add_element/2, add_elements/2, card/1]). + +-record(hll, {value = 0 :: number(), + adds = ordsets:new() :: ordsets:ordset(binary())}). + +-export_type([riakc_hll/0, hll_op/0]). +-opaque riakc_hll() :: #hll{}. + +-type hll_op() :: {add_all, [binary()]}. + +%% @doc Creates a new, empty hll type. +-spec new() -> riakc_hll(). +new() -> + #hll{}. + +%% @doc Creates a new hll +-spec new(riakc_datatype:context()) -> riakc_hll(). +new(_Context) -> + #hll{}. + +%% @doc Creates a new hll +-spec new(number(), riakc_datatype:context()) -> riakc_hll(). +new(Value, _Context) -> + #hll{value=Value}. + +%% @doc Returns the +-spec value(riakc_hll()) -> number(). +value(#hll{value=Value}) -> Value. + +%% @doc Same as value, but better for users. +-spec card(riakc_hll()) -> number(). +card(Hll) -> + value(Hll). + +%% @doc Extracts an operation from the hll that can be encoded into an +%% update request. +-spec to_op(riakc_hll()) -> riakc_datatype:update(hll_op()). +to_op(#hll{adds=[]}) -> + undefined; +to_op(#hll{adds=A}) -> + {type(), {add_all, A}, undefined}. + +%% @doc Determines whether the passed term is a hll type. +-spec is_type(term()) -> boolean(). +is_type(T) -> + is_record(T, hll). + +%% @doc Returns the symbolic name of this type. +-spec type() -> atom(). +type() -> hll. + +%% @doc Adds elements to the hll(set). +-spec add_elements(list(binary()), riakc_hll()) -> riakc_hll(). +add_elements(Elems, Hll) when is_list(Elems) -> + lists:foldl(fun add_element/2, Hll, Elems). + +%% @doc Adds an element to the hll(set). +-spec add_element(binary(), riakc_hll()) -> riakc_hll(). +add_element(Elem, #hll{adds=A0}=Hll) when is_binary(Elem) -> + Hll#hll{adds=ordsets:add_element(Elem, A0)}. + +-ifdef(EQC). +gen_type() -> + ?LET(Elems, list(binary()), new(Elems, undefined)). + +gen_op() -> + oneof([ + {add_element, [binary()]}, + {add_elements, [non_empty(list(binary()))]} + ]). + +-endif. + diff --git a/src/riakc_map.erl b/src/riakc_map.erl index 13f961af..5ed8fdc2 100644 --- a/src/riakc_map.erl +++ b/src/riakc_map.erl @@ -53,7 +53,7 @@ -ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). -include_lib("eunit/include/eunit.hrl"). --compile(export_all). +-export([gen_type/0, gen_op/0]). -endif. %% Callbacks @@ -310,6 +310,6 @@ prop_nested_defaults() -> end). prop_nested_defaults_test() -> - ?assert(eqc:quickcheck(?QC_OUT(prop_nested_defaults()))). + {timeout, 120, [?_assert(eqc:quickcheck(?QC_OUT(prop_nested_defaults())))]}. -endif. diff --git a/src/riakc_pb_socket.erl b/src/riakc_pb_socket.erl index 8a7eecd6..857c06f9 100644 --- a/src/riakc_pb_socket.erl +++ b/src/riakc_pb_socket.erl @@ -4110,6 +4110,31 @@ live_node_tests() -> node = Node, primary = true}], Preflist) + end)}, + {"add redundant and multiple items to hll(set)", + ?_test(begin + reset_riak(), + {ok, Pid} = start_link(test_ip(), test_port()), + ok = update_type(Pid, + {<<"hll_bucket">>, <<"bucket">>}, <<"key">>, + riakc_hll:to_op( + riakc_hll:add_elements([<<"X">>, <<"Y">>], + riakc_hll:new()))), + {ok, Hll0} = fetch_type(Pid, {<<"hll_bucket">>, <<"bucket">>}, + <<"key">>), + ?assertEqual(riakc_hll:value(Hll0), 2), + ok = update_type(Pid, + {<<"hll_bucket">>, <<"bucket">>}, <<"key">>, + riakc_hll:to_op( + riakc_hll:add_element(<<"X">>, Hll0))), + {ok, Hll1} = fetch_type(Pid, {<<"hll_bucket">>, <<"bucket">>}, + <<"key">>), + ?assert(riakc_hll:is_type(Hll1)), + Value = riakc_hll:value(Hll1), + ?assertEqual(Value, 2), + + %% Make sure card and value are the same + ?assertEqual(riakc_hll:card(Hll1), Value) end)} ]. diff --git a/src/riakc_register.erl b/src/riakc_register.erl index b2ca4093..1bd0b5ac 100644 --- a/src/riakc_register.erl +++ b/src/riakc_register.erl @@ -33,7 +33,7 @@ -ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). --compile(export_all). +-export([gen_type/0, gen_op/0]). -endif. %% Callbacks diff --git a/src/riakc_set.erl b/src/riakc_set.erl index 3b9bdc70..a10e9642 100644 --- a/src/riakc_set.erl +++ b/src/riakc_set.erl @@ -49,7 +49,7 @@ -ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). --compile(export_all). +-export([gen_type/0, gen_op/0]). -endif. %% Callbacks @@ -61,6 +61,7 @@ %% Operations -export([add_element/2, + add_elements/2, del_element/2]). %% Query functions @@ -126,6 +127,11 @@ type() -> set. add_element(Bin, #set{adds=A0}=Set) when is_binary(Bin) -> Set#set{adds=ordsets:add_element(Bin, A0)}. +%% @doc Adds elements to the set. +-spec add_elements(list(binary()), riakc_set()) -> riakc_set(). +add_elements(Elems, Set) when is_list(Elems) -> + lists:foldl(fun add_element/2, Set, Elems). + %% @doc Removes an element from the set. %% @throws context_required -spec del_element(binary(), riakc_set()) -> riakc_set(). From 2ba569961a29adbdea460ec50214aafbd13d4fa9 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 15 Apr 2016 16:17:09 -0400 Subject: [PATCH 2/2] update configuration --- buildbot/crdt-setup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildbot/crdt-setup b/buildbot/crdt-setup index 41410e99..155b18fd 100755 --- a/buildbot/crdt-setup +++ b/buildbot/crdt-setup @@ -10,6 +10,7 @@ ADMIN=$1 $ADMIN bucket-type create map_bucket '{"props":{"datatype":"map"}}' || exit 1 $ADMIN bucket-type create set_bucket '{"props":{"datatype":"set"}}' || exit 1 +$ADMIN bucket-type create hll_bucket '{"props":{"datatype":"hll"}}' || exit 1 $ADMIN bucket-type create counter_bucket '{"props":{"datatype":"counter"}}' || exit 1 loop_counter=0 @@ -17,6 +18,7 @@ loop_counter=0 until $ADMIN bucket-type activate map_bucket && \ $ADMIN bucket-type activate set_bucket && \ + $ADMIN bucket-type activate hll_bucket && \ $ADMIN bucket-type activate counter_bucket do echo "Waiting until activation"