[couchdb] branch prototype/fdb-layer updated: Enable configurable binary chunk size

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jaydoane pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/prototype/fdb-layer by this push:
     new 3636451  Enable configurable binary chunk size
3636451 is described below

commit 36364516d32a368b5c58ee197f9c3fbb82394f81
Author: Jay Doane <ja...@apache.org>
AuthorDate: Wed Apr 15 10:19:58 2020 -0700

    Enable configurable binary chunk size
    
    Currently, the size of binary chunks used for values is fixed at the FDB
    imposed limit of 100kB, although they recommend using 10KB [1], (also
    note they subtly change units).
    
    This makes that value configurable, allowing e.g. benchmarks to compare
    performance of runs with varying chunk size. The cost is a ~10µs config
    lookup penalty each time data needs to be chunked.
    
    [1] https://www.foundationdb.org/files/record-layer-paper.pdf
---
 rel/overlay/etc/default.ini    |  3 +++
 src/fabric/include/fabric2.hrl |  2 +-
 src/fabric/src/fabric2_fdb.erl | 15 ++++++++++++---
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index e10a5a0..dfc67f7 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -239,6 +239,9 @@ port = 6984
 ;
 ; Enable or disable automatic stale index removal in the auto-updater
 ;index_updater_remove_old_indices = false
+;
+; Byte size of binary chunks written to FDB values. Defaults to FDB max limit.
+;binary_chunk_size = 100000
 
 ; [rexi]
 ; buffer_count = 2000
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 587b4f8..2e588f8 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -77,4 +77,4 @@
 -define(TRANSACTION_CANCELLED, 1025).
 
 
--define(BINARY_CHUNK_SIZE, 100000).
+-define(DEFAULT_BINARY_CHUNK_SIZE, 100000).
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index d96c3ae..53102d6 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -1628,12 +1628,16 @@ sum_rem_rev_sizes(RevInfos) ->
 
 
 chunkify_binary(Data) ->
+    chunkify_data(Data, binary_chunk_size()).
+
+
+chunkify_data(Data, Size) ->
     case Data of
         <<>> ->
             [];
-        <<Head:?BINARY_CHUNK_SIZE/binary, Rest/binary>> ->
-            [Head | chunkify_binary(Rest)];
-        <<_/binary>> when size(Data) < ?BINARY_CHUNK_SIZE ->
+        <<Head:Size/binary, Rest/binary>> ->
+            [Head | chunkify_data(Rest, Size)];
+        <<_/binary>> when size(Data) < Size ->
             [Data]
     end.
 
@@ -1988,6 +1992,11 @@ get_info_wait_int(#info_future{} = InfoFuture) ->
     [CProp | MProps].
 
 
+binary_chunk_size() ->
+    config:get_integer(
+        "fabric", "binary_chunk_size", ?DEFAULT_BINARY_CHUNK_SIZE).
+
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").