You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/09/10 16:07:39 UTC
[couchdb] 01/01: Merge branch 'master' into
prototype/fdb-layer-final-final-merge
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-final-final-merge
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ec7cfe03f184f38aca6ef577cf0d1cc31a785acc
Merge: 6cedf87 e4d577b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Sep 10 11:01:42 2020 -0500
Merge branch 'master' into prototype/fdb-layer-final-final-merge
.credo.exs | 7 +-
.gitignore | 3 +
INSTALL.Unix.md | 2 +-
LICENSE | 2 +-
Makefile | 29 +-
Makefile.win | 14 +-
README.rst | 8 +-
build-aux/Jenkinsfile.full | 59 ++-
build-aux/logfile-uploader.py | 2 +-
dev/run | 32 +-
mix.exs | 4 +-
mix.lock | 4 +-
rebar.config.script | 23 +-
rel/files/couchdb.cmd.in | 3 +-
rel/overlay/bin/remsh | 21 +-
rel/overlay/etc/default.ini | 30 +-
rel/reltool.config | 4 +-
src/chttpd/src/chttpd_auth.erl | 23 +-
src/chttpd/src/chttpd_db.erl | 1 -
src/chttpd/src/chttpd_misc.erl | 2 +-
src/chttpd/test/eunit/chttpd_auth_tests.erl | 129 ++++++
src/chttpd/test/eunit/chttpd_csp_tests.erl | 2 +-
src/couch/include/couch_eunit.hrl | 5 +
src/couch/priv/couch_js/1.8.5/help.h | 2 +-
src/couch/priv/couch_js/60/help.h | 2 +-
src/couch/priv/couch_js/60/http.cpp | 214 ++++-----
src/couch/priv/couch_js/60/main.cpp | 69 ++-
src/couch/priv/couch_js/60/utf8.cpp | 301 ------------
src/couch/priv/couch_js/60/util.cpp | 196 ++++----
src/couch/priv/couch_js/60/util.h | 4 +-
src/couch/priv/couch_js/{1.8.5 => 68}/help.h | 2 +-
src/couch/priv/couch_js/{60 => 68}/http.cpp | 215 ++++-----
src/couch/priv/couch_js/{60/utf8.h => 68/http.h} | 16 +-
src/couch/priv/couch_js/{60 => 68}/main.cpp | 139 ++++--
src/couch/priv/couch_js/{60 => 68}/util.cpp | 205 +++++----
src/couch/priv/couch_js/{60 => 68}/util.h | 5 +-
src/couch/rebar.config.script | 72 +--
src/couch/src/couch.app.src | 1 -
src/couch/src/couch.erl | 1 -
src/couch/src/couch_httpd.erl | 2 +
src/couch/src/couch_httpd_auth.erl | 41 +-
src/couch/src/couch_query_servers.erl | 107 ++++-
src/couch/src/couch_server.erl | 15 +-
src/couch/src/couch_util.erl | 12 +
src/couch/test/eunit/couch_js_tests.erl | 139 +++++-
.../test/eunit/couchdb_cookie_domain_tests.erl | 13 +-
src/couch_index/src/couch_index_server.erl | 16 +-
src/couch_mrview/src/couch_mrview_index.erl | 11 +-
src/couch_mrview/src/couch_mrview_util.erl | 9 +-
src/couch_replicator/src/couch_replicator.erl | 28 +-
.../src/couch_replicator_api_wrap.erl | 2 +-
.../src/couch_replicator_scheduler.erl | 16 +
src/dreyfus/src/dreyfus_fabric_cleanup.erl | 16 +-
src/dreyfus/src/dreyfus_httpd.erl | 24 +-
.../test/elixir/test/partition_search_test.exs | 36 +-
src/dreyfus/test/elixir/test/search_test.exs | 226 +++++++++
src/fabric/src/fabric_rpc.erl | 5 +-
src/fabric/test/eunit/fabric_rpc_tests.erl | 181 ++++++++
src/ioq/src/ioq.erl | 2 +-
src/jwtf/.gitignore | 4 +
src/jwtf/LICENSE | 176 +++++++
src/jwtf/README.md | 18 +
src/jwtf/rebar.config | 2 +
src/jwtf/src/jwtf.app.src | 32 ++
src/jwtf/src/jwtf.erl | 353 ++++++++++++++
src/jwtf/src/jwtf_app.erl | 28 ++
src/jwtf/src/jwtf_keystore.erl | 166 +++++++
src/jwtf/src/jwtf_sup.erl | 38 ++
src/jwtf/test/jwtf_keystore_tests.erl | 64 +++
src/jwtf/test/jwtf_tests.erl | 317 +++++++++++++
src/mango/README.md | 328 +++++++++----
src/mango/TODO.md | 19 +-
src/mango/rebar.config | 2 +
src/mango/src/mango_cursor_text.erl | 17 +-
src/mango/src/mango_cursor_view.erl | 5 +-
src/mango/src/mango_json_bookmark.erl | 2 +-
src/mango/src/mango_selector.erl | 36 +-
src/mango/test/03-operator-test.py | 9 +
src/mango/test/08-text-limit-test.py | 10 +
src/mango/test/21-empty-selector-tests.py | 24 +-
src/mango/test/README.md | 17 +-
src/mango/test/mango.py | 8 +
src/mango/test/user_docs.py | 4 +
src/mem3/src/mem3_reshard_dbdoc.erl | 3 +-
src/mem3/src/mem3_rpc.erl | 2 +-
src/mem3/src/mem3_shards.erl | 22 +-
src/mem3/src/mem3_util.erl | 86 +++-
src/setup/src/setup.erl | 5 +-
support/build_js.escript | 6 +
test/elixir/README.md | 53 +--
test/elixir/lib/couch.ex | 4 +-
test/elixir/lib/couch/db_test.ex | 83 +++-
test/elixir/test/all_docs_test.exs | 1 +
test/elixir/test/attachment_names_test.exs | 1 +
test/elixir/test/attachment_paths_test.exs | 1 +
test/elixir/test/attachment_ranges_test.exs | 1 +
test/elixir/test/attachment_views_test.exs | 1 +
test/elixir/test/attachments_multipart_test.exs | 1 +
test/elixir/test/attachments_test.exs | 1 +
test/elixir/test/auth_cache_test.exs | 16 +-
test/elixir/test/basics_test.exs | 1 +
test/elixir/test/batch_save_test.exs | 1 +
test/elixir/test/bulk_docs_test.exs | 1 +
test/elixir/test/changes_async_test.exs | 1 +
test/elixir/test/changes_test.exs | 1 +
test/elixir/test/cluster_with_quorum_test.exs | 1 +
test/elixir/test/cluster_without_quorum_test.exs | 1 +
test/elixir/test/coffee_test.exs | 1 +
test/elixir/test/compact_test.exs | 13 +-
test/elixir/test/config/test-config.ini | 2 +
test/elixir/test/config_test.exs | 1 +
test/elixir/test/conflicts_test.exs | 1 +
test/elixir/test/cookie_auth_test.exs | 16 +-
test/elixir/test/copy_doc_test.exs | 1 +
test/elixir/test/design_docs_query_test.exs | 274 +++++++++++
test/elixir/test/design_docs_test.exs | 508 ++++++++++++++++++---
test/elixir/test/design_options_test.exs | 75 +++
test/elixir/test/design_paths_test.exs | 77 ++++
test/elixir/test/erlang_views_test.exs | 1 +
test/elixir/test/etags_head_test.exs | 1 +
test/elixir/test/form_submit_test.exs | 1 +
test/elixir/test/helper_test.exs | 3 +
test/elixir/test/http_test.exs | 82 ++++
test/elixir/test/invalid_docids_test.exs | 1 +
test/elixir/test/jsonp_test.exs | 117 +++++
test/elixir/test/jwtauth_test.exs | 218 +++++++++
test/elixir/test/large_docs_text.exs | 2 +
test/elixir/test/local_docs_test.exs | 1 +
test/elixir/test/lots_of_docs_test.exs | 2 +
test/elixir/test/method_override_test.exs | 56 +++
test/elixir/test/multiple_rows_test.exs | 1 +
test/elixir/test/proxyauth_test.exs | 164 +++++++
test/elixir/test/purge_test.exs | 151 ++++++
test/elixir/test/reader_acl_test.exs | 255 +++++++++++
test/elixir/test/recreate_doc_test.exs | 166 +++++++
test/elixir/test/reduce_builtin_test.exs | 1 +
test/elixir/test/reduce_false_test.exs | 1 +
test/elixir/test/reduce_test.exs | 1 +
test/elixir/test/replication_test.exs | 41 +-
test/elixir/test/replicator_db_bad_rep_id_test.exs | 84 ++++
test/elixir/test/replicator_db_by_doc_id_test.exs | 124 +++++
test/elixir/test/rev_stemming_test.exs | 158 +++++++
test/elixir/test/rewrite_test.exs | 1 +
test/elixir/test/security_validation_test.exs | 1 +
test/elixir/test/update_documents_test.exs | 326 +++++++++++++
test/elixir/test/users_db_test.exs | 26 +-
test/elixir/test/utf8_test.exs | 3 +-
test/elixir/test/uuids_test.exs | 3 +
...lation_test.exs => view_collation_raw_test.exs} | 111 +++--
test/elixir/test/view_collation_test.exs | 2 +
test/elixir/test/view_compaction_test.exs | 108 +++++
test/elixir/test/view_multi_key_all_docs_test.exs | 193 ++++++++
test/elixir/test/view_multi_key_design_test.exs | 318 +++++++++++++
test/elixir/test/view_offsets_test.exs | 101 ++++
test/elixir/test/view_pagination_test.exs | 190 ++++++++
test/elixir/test/view_sandboxing_test.exs | 193 ++++++++
test/elixir/test/view_test.exs | 1 +
test/elixir/test/view_update_seq_test.exs | 143 ++++++
test/javascript/cli_runner.js | 23 +-
test/javascript/tests/changes.js | 6 +-
test/javascript/tests/design_docs.js | 2 +
test/javascript/tests/design_docs_query.js | 4 +-
test/javascript/tests/design_options.js | 5 +-
test/javascript/tests/design_paths.js | 3 +-
test/javascript/tests/erlang_views.js | 2 +-
test/javascript/tests/form_submit.js | 1 +
test/javascript/tests/http.js | 3 +-
test/javascript/tests/jsonp.js | 2 +
test/javascript/tests/method_override.js | 2 +
test/javascript/tests/proxyauth.js | 51 +--
test/javascript/tests/purge.js | 2 +-
test/javascript/tests/reader_acl.js | 1 +
test/javascript/tests/recreate_doc.js | 1 +
test/javascript/tests/reduce_builtin.js | 1 +
test/javascript/tests/reduce_false.js | 1 +
test/javascript/tests/reduce_false_temp.js | 1 +
test/javascript/tests/replicator_db_bad_rep_id.js | 1 +
test/javascript/tests/replicator_db_by_doc_id.js | 1 +
test/javascript/tests/rev_stemming.js | 1 +
test/javascript/tests/update_documents.js | 2 +-
test/javascript/tests/view_collation_raw.js | 1 +
test/javascript/tests/view_compaction.js | 1 +
test/javascript/tests/view_multi_key_all_docs.js | 1 +
test/javascript/tests/view_multi_key_design.js | 1 +
test/javascript/tests/view_multi_key_temp.js | 1 +
test/javascript/tests/view_offsets.js | 2 +
test/javascript/tests/view_pagination.js | 2 +
test/javascript/tests/view_sandboxing.js | 1 +
test/javascript/tests/view_update_seq.js | 1 +
189 files changed, 7797 insertions(+), 1378 deletions(-)
diff --cc .gitignore
index 5c42552,6223d73..ead569d
--- a/.gitignore
+++ b/.gitignore
@@@ -11,10 -11,10 +11,12 @@@
.DS_Store
.vscode
.rebar/
+.rebar3/
+.erlfdb/
.eunit/
cover/
+ core
+ debian/
log
apache-couchdb-*/
bin/
diff --cc Makefile
index 2e3cc8a,53cea3b..ae095b6
--- a/Makefile
+++ b/Makefile
@@@ -144,11 -144,10 +144,12 @@@ fauxton: share/ww
################################################################################
-.PHONY: check
+# When we can run all the tests with FDB switch this back to be the default
+# "make check" command
+.PHONY: check-all-tests
# target: check - Test everything
-check: all python-black
+check-all-tests: all python-black
+ @$(MAKE) emilio
@$(MAKE) eunit
@$(MAKE) javascript
@$(MAKE) mango-test
@@@ -237,11 -227,10 +238,25 @@@ python-black-update: .venv/bin/blac
elixir: export MIX_ENV=integration
elixir: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
elixir: elixir-init elixir-check-formatted elixir-credo devclean
- @dev/run "$(TEST_OPTS)" -a adm:pass -n 1 --enable-erlang-views --no-eval --erlang-config=rel/files/eunit.config 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
- @dev/run "$(TEST_OPTS)" -a adm:pass -n 1 \
++ @dev/run "$(TEST_OPTS)" \
++ -a adm:pass \
++ -n 1 \
++ --enable-erlang-views \
++ --locald-config test/elixir/test/config/test-config.ini \
++ --erlang-config rel/files/eunit.config \
++ --no-eval \
++ 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
+
+.PHONY: elixir-only
+elixir-only: devclean
- @dev/run "$(TEST_OPTS)" -a adm:pass -n 1 --enable-erlang-views --no-eval --erlang-config=rel/files/eunit.config 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
++ @dev/run "$(TEST_OPTS)" \
++ -a adm:pass \
++ -n 1 \
+ --enable-erlang-views \
+ --locald-config test/elixir/test/config/test-config.ini \
- --no-eval 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
++ --erlang-config rel/files/eunit.config \
++ --no-eval \
++ 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
.PHONY: elixir-init
elixir-init: MIX_ENV=test
diff --cc mix.exs
index 480d426,ae42af5..5adccb7
--- a/mix.exs
+++ b/mix.exs
@@@ -68,11 -65,12 +68,13 @@@ defmodule CouchDBTest.Mixfile d
{:junit_formatter, "~> 3.0", only: [:dev, :test, :integration]},
{:httpotion, ">= 3.1.3", only: [:dev, :test, :integration], runtime: false},
{:excoveralls, "~> 0.12", only: :test},
+ {:b64url, path: Path.expand("src/b64url", __DIR__)},
{:jiffy, path: Path.expand("src/jiffy", __DIR__)},
+ {:jwtf, path: Path.expand("src/jwtf", __DIR__)},
{:ibrowse,
path: Path.expand("src/ibrowse", __DIR__), override: true, compile: false},
- {:credo, "~> 1.2.0", only: [:dev, :test, :integration], runtime: false},
- {:credo, "~> 1.4.0", only: [:dev, :test, :integration], runtime: false}
++ {:credo, "~> 1.4.0", only: [:dev, :test, :integration], runtime: false},
+ {:stream_data, "~> 0.4.3", only: [:dev, :test, :integration], runtime: false}
]
end
diff --cc rebar.config.script
index 963d97f,d8afc10..9947944
--- a/rebar.config.script
+++ b/rebar.config.script
@@@ -135,19 -130,16 +135,20 @@@ SubDirs =
"src/ddoc_cache",
"src/dreyfus",
"src/fabric",
+ "src/aegis",
+ "src/couch_jobs",
+ "src/couch_expiring_cache",
"src/global_changes",
"src/ioq",
+ "src/jwtf",
"src/ken",
"src/mango",
"src/rexi",
"src/setup",
"src/smoosh",
+ "src/ebtree",
"rel"
- ],
+ ].
DepDescs = [
%% Independent Apps
@@@ -166,18 -157,14 +167,18 @@@
%% Third party deps
{folsom, "folsom", {tag, "CouchDB-0.8.3"}},
{hyper, "hyper", {tag, "CouchDB-2.2.0-6"}},
-{ibrowse, "ibrowse", {tag, "CouchDB-4.0.1-1"}},
+{ibrowse, "ibrowse", {tag, "CouchDB-4.0.1-2"}},
+{jaeger_passage, "jaeger-passage", {tag, "CouchDB-0.1.14-1"}},
{jiffy, "jiffy", {tag, "CouchDB-1.0.4-1"}},
+{local, "local", {tag, "0.2.1"}},
{mochiweb, "mochiweb", {tag, "v2.20.0"}},
{meck, "meck", {tag, "0.8.8"}},
-{recon, "recon", {tag, "2.5.0"}}
+{recon, "recon", {tag, "2.5.0"}},
+{passage, "passage", {tag, "0.2.6"}},
+{thrift_protocol, "thrift-protocol", {tag, "0.1.5"}}
- ],
+ ].
- WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true},
+ WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}.
OptionalDeps = case WithProper of
true ->
@@@ -223,13 -210,8 +224,13 @@@ AddConfig =
sasl, setup, ssl, stdlib, syntax_tools, xmerl]},
{warnings, [unmatched_returns, error_handling, race_conditions]}]},
{post_hooks, [{compile, "escript support/build_js.escript"}]}
- ],
+ ].
-C = lists:foldl(fun({K, V}, CfgAcc) ->
- lists:keystore(K, 1, CfgAcc, {K, V})
+lists:foldl(fun({K, V}, CfgAcc) ->
+ case lists:keyfind(K, 1, CfgAcc) of
+ {K, Existent} when is_list(Existent) andalso is_list(V) ->
+ lists:keystore(K, 1, CfgAcc, {K, Existent ++ V});
+ false ->
+ lists:keystore(K, 1, CfgAcc, {K, V})
+ end
end, CONFIG, AddConfig).
diff --cc src/chttpd/src/chttpd_db.erl
index ec4a1a4,b995460..18d663f
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@@ -1865,7 -1695,81 +1865,6 @@@ get_md5_header(Req) -
parse_doc_query(Req) ->
lists:foldl(fun parse_doc_query/2, #doc_query_args{}, chttpd:qs(Req)).
-parse_shards_opt(Req) ->
- [
- {n, parse_shards_opt("n", Req, config:get("cluster", "n", "3"))},
- {q, parse_shards_opt("q", Req, config:get("cluster", "q", "8"))},
- {placement, parse_shards_opt(
- "placement", Req, config:get("cluster", "placement"))}
- ].
-
-parse_shards_opt("placement", Req, Default) ->
- Err = <<"The `placement` value should be in a format `zone:n`.">>,
- case chttpd:qs_value(Req, "placement", Default) of
- Default -> Default;
- [] -> throw({bad_request, Err});
- Val ->
- try
- true = lists:all(fun(Rule) ->
- [_, N] = string:tokens(Rule, ":"),
- couch_util:validate_positive_int(N)
- end, string:tokens(Val, ",")),
- Val
- catch _:_ ->
- throw({bad_request, Err})
- end
- end;
-
-parse_shards_opt(Param, Req, Default) ->
- Val = chttpd:qs_value(Req, Param, Default),
- Err = ?l2b(["The `", Param, "` value should be a positive integer."]),
- case couch_util:validate_positive_int(Val) of
- true -> Val;
- false -> throw({bad_request, Err})
- end.
-
-
-parse_engine_opt(Req) ->
- case chttpd:qs_value(Req, "engine") of
- undefined ->
- [];
- Extension ->
- Available = couch_server:get_engine_extensions(),
- case lists:member(Extension, Available) of
- true ->
- [{engine, iolist_to_binary(Extension)}];
- false ->
- throw({bad_request, invalid_engine_extension})
- end
- end.
-
-
-parse_partitioned_opt(Req) ->
- case chttpd:qs_value(Req, "partitioned") of
- undefined ->
- [];
- "false" ->
- [];
- "true" ->
- ok = validate_partitioned_db_enabled(Req),
- [
- {partitioned, true},
- {hash, [couch_partition, hash, []]}
- ];
- _ ->
- throw({bad_request, <<"Invalid `partitioned` parameter">>})
- end.
-
-
-validate_partitioned_db_enabled(Req) ->
- case couch_flags:is_enabled(partitioned, Req) of
- true ->
- ok;
- false ->
- throw({bad_request, <<"Partitioned feature is not enabled.">>})
- end.
-
--
parse_doc_query({Key, Value}, Args) ->
case {Key, Value} of
{"attachments", "true"} ->
diff --cc src/mango/src/mango_cursor_view.erl
index 4960fa1,68d7c3b..44ae220
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@@ -112,7 -115,12 +112,10 @@@ base_args(#cursor{index = Idx} = Cursor
reduce = false,
start_key = StartKey,
end_key = EndKey,
- include_docs = true
+ include_docs = true,
+ extra = [
- {callback, {?MODULE, view_cb}},
- {selector, Selector},
+ {ignore_partition_query_limit, true}
+ ]
}.
diff --cc src/mango/test/user_docs.py
index d69e6d6,617b430..c301983
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@@ -59,8 -59,13 +59,12 @@@ def setup_users(db, **kwargs)
db.save_docs(copy.deepcopy(USERS_DOCS))
+ def teardown_users(db):
+ [db.delete_doc(doc["_id"]) for doc in USERS_DOCS]
+
+
def setup(db, index_type="view", **kwargs):
db.recreate()
- db.save_docs(copy.deepcopy(DOCS))
if index_type == "view":
add_view_indexes(db, kwargs)
elif index_type == "text":
diff --cc test/elixir/lib/couch.ex
index a843941,7819299..094f275
--- a/test/elixir/lib/couch.ex
+++ b/test/elixir/lib/couch.ex
@@@ -129,15 -125,10 +129,15 @@@ defmodule Couch d
end
def set_auth_options(options) do
- if Keyword.get(options, :cookie) == nil do
+ no_auth? = Keyword.get(options, :no_auth) == true
+ cookie? = Keyword.has_key?(options, :cookie)
+ basic_auth? = Keyword.has_key?(options, :basic_auth)
+ if cookie? or no_auth? or basic_auth? do
+ Keyword.delete(options, :no_auth)
+ else
headers = Keyword.get(options, :headers, [])
-
- if headers[:basic_auth] != nil or headers[:authorization] != nil do
+ if headers[:basic_auth] != nil or headers[:authorization] != nil
+ or List.keymember?(headers, :"X-Auth-CouchDB-UserName", 0) do
options
else
username = System.get_env("EX_USERNAME") || "adm"
diff --cc test/elixir/test/replication_test.exs
index 8b657d9,7b462bd..01694a7
--- a/test/elixir/test/replication_test.exs
+++ b/test/elixir/test/replication_test.exs
@@@ -1804,29 -1706,15 +1782,16 @@@ defmodule ReplicationTest d
end
def try_get_task(repl_id) do
- resp = Couch.get("/_active_tasks")
- assert HTTPotion.Response.success?(resp)
- assert is_list(resp.body)
+ resp = Couch.get("/_scheduler/jobs/#{repl_id}")
- Enum.find(resp.body, nil, fn task ->
- task["replication_id"] == repl_id
- end)
+ if HTTPotion.Response.success?(resp) do
+ assert is_map(resp.body)
+ resp.body
+ else
+ nil
+ end
end
- def set_user(uri, userinfo) do
- case URI.parse(uri) do
- %{scheme: nil} ->
- uri
-
- %{userinfo: nil} = uri ->
- URI.to_string(Map.put(uri, :userinfo, userinfo))
-
- _ ->
- uri
- end
- end
-
def get_att1_data do
File.read!(Path.expand("data/lorem.txt", __DIR__))
end