Merge pull request #268 from basho/feature-zl-hll_datatypes

add Hyperloglog DT support and few minor test fixins

buildbot/crdt-setup

@@ -10,13 +10,15 @@ ADMIN=$1
 
 $ADMIN bucket-type create map_bucket '{"props":{"datatype":"map"}}' || exit 1
 $ADMIN bucket-type create set_bucket '{"props":{"datatype":"set"}}' || exit 1
+$ADMIN bucket-type create hll_bucket '{"props":{"datatype":"hll"}}' || exit 1
 $ADMIN bucket-type create counter_bucket '{"props":{"datatype":"counter"}}' || exit 1
 
 loop_counter=0
 
 until
     $ADMIN bucket-type activate map_bucket && \
     $ADMIN bucket-type activate set_bucket && \
+    $ADMIN bucket-type activate hll_bucket && \
     $ADMIN bucket-type activate counter_bucket
 do
     echo "Waiting until activation"

rebar.config

@@ -13,7 +13,7 @@
 ]}.
 
 {deps, [
-    {riak_pb, "2.1.*", {git, "https://github.com/basho/riak_pb", {tag, "2.1.4.1"}}}
+    {riak_pb, ".*", {git, "git://github.com/basho/riak_pb", {branch, "feature-zl-hll_datatypes"}}}
 ]}.
 
 {edoc_opts, [

src/riakc_counter.erl

@@ -31,7 +31,7 @@
 
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
--compile(export_all).
+-export([gen_type/0, gen_op/0]).
 -endif.
 
 %% Callbacks

src/riakc_datatype.erl

@@ -2,7 +2,7 @@
 %%
 %% riakc_datatype: Behaviour for eventually-consistent data-types
 %%
-%% Copyright (c) 2013 Basho Technologies, Inc.  All Rights Reserved.
+%% Copyright (c) 2016 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
@@ -30,11 +30,14 @@
 -include_lib("eqc/include/eqc.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -define(QC_OUT(P), eqc:on_output(fun(Fmt, Args) -> io:format(user, Fmt, Args) end, P)).
+-callback gen_type() -> eqc_gen:gen(datatype()).
+-callback gen_op()   -> eqc_gen:gen({atom(), [term()]}).
 -compile(export_all).
 -endif.
 
 
--define(MODULES, [riakc_set, riakc_counter, riakc_flag, riakc_register, riakc_map]).
+-define(MODULES, [riakc_set, riakc_counter, riakc_flag, riakc_register,
+                  riakc_map, riakc_hll]).
 
 -export([module_for_type/1,
          module_for_term/1]).
@@ -82,6 +85,7 @@
 %% type.
 -spec module_for_type(Type::atom()) -> module().
 module_for_type(set)      -> riakc_set;
+module_for_type(hll)      -> riakc_hll;
 module_for_type(counter)  -> riakc_counter;
 module_for_type(flag)     -> riakc_flag;
 module_for_type(register) -> riakc_register;
@@ -109,9 +113,9 @@ module_for_term(T) ->
 -define(F(Fmt, Args), lists:flatten(io_lib:format(Fmt, Args))).
 datatypes_test_() ->
      [{" prop_module_type() ",
-       ?_assertEqual(true, quickcheck(?QC_OUT(prop_module_type())))}] ++
+       ?_assert(eqc:quickcheck(?QC_OUT(prop_module_type())))}] ++
      [ {?F(" ~s(~s) ", [Prop, Mod]),
-        ?_assertEqual(true, quickcheck(?QC_OUT(eqc:testing_time(2, ?MODULE:Prop(Mod)))))} ||
+        ?_assert(eqc:quickcheck(?QC_OUT(eqc:testing_time(2, ?MODULE:Prop(Mod)))))} ||
          Prop <- ?MODPROPS,
          Mod <- ?MODULES ].
 

src/riakc_flag.erl

@@ -33,7 +33,7 @@
 
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
--compile(export_all).
+-export([gen_type/0, gen_op/0]).
 -endif.
 
 %% Callbacks

src/riakc_hll.erl

@@ -0,0 +1,113 @@
+%% -------------------------------------------------------------------
+%%
+%% riakc_set: Eventually-consistent hyperloglog(set) type
+%%
+%% Copyright (c) 2013 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% -------------------------------------------------------------------
+
+
+%%@doc
+-module(riakc_hll).
+-behaviour(riakc_datatype).
+
+-ifdef(EQC).
+-include_lib("eqc/include/eqc.hrl").
+-export([gen_type/0, gen_op/0]).
+-endif.
+
+%% Callbacks
+-export([new/0, new/1, new/2,
+         value/1,
+         to_op/1,
+         is_type/1,
+         type/0]).
+
+%% Operations
+-export([add_element/2, add_elements/2, card/1]).
+
+-record(hll, {value = 0 :: number(),
+              adds = ordsets:new() :: ordsets:ordset(binary())}).
+
+-export_type([riakc_hll/0, hll_op/0]).
+-opaque riakc_hll() :: #hll{}.
+
+-type hll_op() :: {add_all, [binary()]}.
+
+%% @doc Creates a new, empty hll type.
+-spec new() -> riakc_hll().
+new() ->
+    #hll{}.
+
+%% @doc Creates a new hll
+-spec new(riakc_datatype:context()) -> riakc_hll().
+new(_Context) ->
+    #hll{}.
+
+%% @doc Creates a new hll
+-spec new(number(), riakc_datatype:context()) -> riakc_hll().
+new(Value, _Context) ->
+    #hll{value=Value}.
+
+%% @doc Returns the
+-spec value(riakc_hll()) -> number().
+value(#hll{value=Value}) -> Value.
+
+%% @doc Same as value, but better for users.
+-spec card(riakc_hll()) -> number().
+card(Hll) ->
+    value(Hll).
+
+%% @doc Extracts an operation from the hll that can be encoded into an
+%% update request.
+-spec to_op(riakc_hll()) -> riakc_datatype:update(hll_op()).
+to_op(#hll{adds=[]}) ->
+    undefined;
+to_op(#hll{adds=A}) ->
+    {type(), {add_all, A}, undefined}.
+
+%% @doc Determines whether the passed term is a hll type.
+-spec is_type(term()) -> boolean().
+is_type(T) ->
+    is_record(T, hll).
+
+%% @doc Returns the symbolic name of this type.
+-spec type() -> atom().
+type() -> hll.
+
+%% @doc Adds elements to the hll(set).
+-spec add_elements(list(binary()), riakc_hll()) -> riakc_hll().
+add_elements(Elems, Hll) when is_list(Elems) ->
+    lists:foldl(fun add_element/2, Hll, Elems).
+
+%% @doc Adds an element to the hll(set).
+-spec add_element(binary(), riakc_hll()) -> riakc_hll().
+add_element(Elem, #hll{adds=A0}=Hll) when is_binary(Elem) ->
+    Hll#hll{adds=ordsets:add_element(Elem, A0)}.
+
+-ifdef(EQC).
+gen_type() ->
+    ?LET(Elems, list(binary()), new(Elems, undefined)).
+
+gen_op() ->
+    oneof([
+           {add_element, [binary()]},
+           {add_elements, [non_empty(list(binary()))]}
+          ]).
+
+-endif.
+

src/riakc_map.erl

@@ -53,7 +53,7 @@
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
 -include_lib("eunit/include/eunit.hrl").
--compile(export_all).
+-export([gen_type/0, gen_op/0]).
 -endif.
 
 %% Callbacks
@@ -310,6 +310,6 @@ prop_nested_defaults() ->
             end).
 
 prop_nested_defaults_test() ->
-    ?assert(eqc:quickcheck(?QC_OUT(prop_nested_defaults()))).
+    {timeout, 120, [?_assert(eqc:quickcheck(?QC_OUT(prop_nested_defaults())))]}.
 
 -endif.

src/riakc_pb_socket.erl

@@ -4110,6 +4110,31 @@ live_node_tests() ->
                                               node = Node,
                                               primary = true}],
                               Preflist)
+             end)},
+     {"add redundant and multiple items to hll(set)",
+      ?_test(begin
+                 reset_riak(),
+                 {ok, Pid} = start_link(test_ip(), test_port()),
+                 ok = update_type(Pid,
+                                  {<<"hll_bucket">>, <<"bucket">>}, <<"key">>,
+                                  riakc_hll:to_op(
+                                    riakc_hll:add_elements([<<"X">>, <<"Y">>],
+                                                          riakc_hll:new()))),
+                 {ok, Hll0} = fetch_type(Pid, {<<"hll_bucket">>, <<"bucket">>},
+                                       <<"key">>),
+                 ?assertEqual(riakc_hll:value(Hll0), 2),
+                 ok = update_type(Pid,
+                                  {<<"hll_bucket">>, <<"bucket">>}, <<"key">>,
+                                  riakc_hll:to_op(
+                                    riakc_hll:add_element(<<"X">>, Hll0))),
+                 {ok, Hll1} = fetch_type(Pid, {<<"hll_bucket">>, <<"bucket">>},
+                                         <<"key">>),
+                 ?assert(riakc_hll:is_type(Hll1)),
+                 Value = riakc_hll:value(Hll1),
+                 ?assertEqual(Value, 2),
+
+                 %% Make sure card and value are the same
+                 ?assertEqual(riakc_hll:card(Hll1), Value)
              end)}
      ].
 

src/riakc_register.erl

@@ -33,7 +33,7 @@
 
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
--compile(export_all).
+-export([gen_type/0, gen_op/0]).
 -endif.
 
 %% Callbacks

src/riakc_set.erl

@@ -49,7 +49,7 @@
 
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
--compile(export_all).
+-export([gen_type/0, gen_op/0]).
 -endif.
 
 %% Callbacks
@@ -61,6 +61,7 @@
 
 %% Operations
 -export([add_element/2,
+         add_elements/2,
          del_element/2]).
 
 %% Query functions
@@ -126,6 +127,11 @@ type() -> set.
 add_element(Bin, #set{adds=A0}=Set) when is_binary(Bin) ->
     Set#set{adds=ordsets:add_element(Bin, A0)}.
 
+%% @doc Adds elements to the set.
+-spec add_elements(list(binary()), riakc_set()) -> riakc_set().
+add_elements(Elems, Set) when is_list(Elems) ->
+    lists:foldl(fun add_element/2, Set, Elems).
+
 %% @doc Removes an element from the set.
 %% @throws context_required
 -spec del_element(binary(), riakc_set()) -> riakc_set().