You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2020/04/10 18:33:54 UTC
[couchdb] branch configurable-binary-chunk-size created (now
f70bf20)
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a change to branch configurable-binary-chunk-size
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at f70bf20 Enable configurable binary chunk size
This branch includes the following new commits:
new f70bf20 Enable configurable binary chunk size
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Enable configurable binary chunk size
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch configurable-binary-chunk-size
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f70bf20fbb75c018d798cb23c1b9d4fbc399d9d6
Author: Jay Doane <ja...@apache.org>
AuthorDate: Thu Apr 9 17:20:58 2020 -0700
Enable configurable binary chunk size
It might be useful to compare performance benchmarks which vary the
binary chunk size from the maximum supported by FDB.
---
src/fabric/include/fabric2.hrl | 2 +-
src/fabric/src/fabric2_fdb.erl | 31 +++++++++++++++++++++----------
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 587b4f8..2e588f8 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -77,4 +77,4 @@
-define(TRANSACTION_CANCELLED, 1025).
--define(BINARY_CHUNK_SIZE, 100000).
+-define(DEFAULT_BINARY_CHUNK_SIZE, 100000).
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index d96c3ae..c304b77 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -235,6 +235,7 @@ create(#{} = Db0, Options) ->
validate_doc_update_funs => [],
before_doc_update => undefined,
after_doc_read => undefined,
+ binary_chunk_size => binary_chunk_size(),
% All other db things as we add features,
db_options => Options1
@@ -278,6 +279,7 @@ open(#{} = Db0, Options) ->
validate_doc_update_funs => [],
before_doc_update => undefined,
after_doc_read => undefined,
+ binary_chunk_size => binary_chunk_size(),
db_options => Options2
},
@@ -996,7 +998,8 @@ write_attachment(#{} = Db, DocId, Data, Encoding)
when is_binary(Data), is_atom(Encoding) ->
#{
tx := Tx,
- db_prefix := DbPrefix
+ db_prefix := DbPrefix,
+ binary_chunk_size := BinaryChunkSize
} = ensure_current(Db),
AttId = fabric2_util:uuid(),
@@ -1017,7 +1020,7 @@ write_attachment(#{} = Db, DocId, Data, Encoding)
InfoVal = erlfdb_tuple:pack({?CURR_ATT_STORAGE_VER, Compressed}),
ok = erlfdb:set(Tx, IdKey, InfoVal),
- Chunks = chunkify_binary(Data1),
+ Chunks = chunkify_binary(Data1, BinaryChunkSize),
lists:foldl(fun(Chunk, ChunkId) ->
AttKey = erlfdb_tuple:pack({?DB_ATTS, DocId, AttId, ChunkId}, DbPrefix),
@@ -1488,7 +1491,8 @@ fdb_to_revinfo(Key, {1, RPath, AttHash}) ->
doc_to_fdb(Db, #doc{} = Doc) ->
#{
- db_prefix := DbPrefix
+ db_prefix := DbPrefix,
+ binary_chunk_size := BinaryChunkSize
} = Db,
#doc{
@@ -1503,7 +1507,7 @@ doc_to_fdb(Db, #doc{} = Doc) ->
Opts = [{minor_version, 1}, {compressed, 6}],
Value = term_to_binary({Body, DiskAtts, Deleted}, Opts),
- Chunks = chunkify_binary(Value),
+ Chunks = chunkify_binary(Value, BinaryChunkSize),
{Rows, _} = lists:mapfoldl(fun(Chunk, ChunkId) ->
Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev, ChunkId}, DbPrefix),
@@ -1538,7 +1542,8 @@ fdb_to_doc(Db, DocId, Pos, Path, BinRows) when is_list(BinRows) ->
local_doc_to_fdb(Db, #doc{} = Doc) ->
#{
- db_prefix := DbPrefix
+ db_prefix := DbPrefix,
+ binary_chunk_size := BinaryChunkSize
} = Db,
#doc{
@@ -1558,7 +1563,7 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
{Rows, _} = lists:mapfoldl(fun(Chunk, ChunkId) ->
K = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, Id, ChunkId}, DbPrefix),
{{K, Chunk}, ChunkId + 1}
- end, 0, chunkify_binary(BVal)),
+ end, 0, chunkify_binary(BVal, BinaryChunkSize)),
NewSize = fabric2_util:ldoc_size(Doc),
RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, NewSize}),
@@ -1627,13 +1632,14 @@ sum_rem_rev_sizes(RevInfos) ->
end, 0, RevInfos).
-chunkify_binary(Data) ->
+chunkify_binary(Data, BinaryChunkSize) ->
+ couch_log:info("chunkify_binary ~p", [BinaryChunkSize]),
case Data of
<<>> ->
[];
- <<Head:?BINARY_CHUNK_SIZE/binary, Rest/binary>> ->
- [Head | chunkify_binary(Rest)];
- <<_/binary>> when size(Data) < ?BINARY_CHUNK_SIZE ->
+ <<Head:BinaryChunkSize/binary, Rest/binary>> ->
+ [Head | chunkify_binary(Rest, BinaryChunkSize)];
+ <<_/binary>> when size(Data) < BinaryChunkSize ->
[Data]
end.
@@ -1988,6 +1994,11 @@ get_info_wait_int(#info_future{} = InfoFuture) ->
[CProp | MProps].
+binary_chunk_size() ->
+ config:get_integer(
+ "fabric", "binary_chunk_size", ?DEFAULT_BINARY_CHUNK_SIZE).
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").