You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2023/06/06 22:32:19 UTC
[couchdb] branch main updated: Add optional logging of security issues when replicating (#4625)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 604526f5f Add optional logging of security issues when replicating (#4625)
604526f5f is described below
commit 604526f5f93df28138a165a666e39ff37f3fdc06
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Jun 6 22:32:12 2023 +0000
Add optional logging of security issues when replicating (#4625)
Add optional logging of security issues when replicating
---
rel/overlay/etc/default.ini | 6 ++
src/couch_replicator/src/couch_replicator.erl | 1 +
.../src/couch_replicator_doc_processor.erl | 1 +
.../src/couch_replicator_scheduler_job.erl | 16 ++-
.../src/couch_replicator_utils.erl | 108 ++++++++++++++++++++-
src/docs/src/config/replicator.rst | 38 ++++++--
6 files changed, 160 insertions(+), 10 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 4f2c44d95..2903e7603 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -589,6 +589,12 @@ partitioned||* = true
; in this list will fail to run.
;valid_endpoint_protocols = http,https
+; When enabled CouchDB will log any replication that uses the insecure http protocol.
+;valid_endpoint_protocols_log = false
+
+; When enabled CouchDB will check the validity of the TLS certificates of source and target.
+;verify_ssl_certificates_log = false
+
; Valid replication proxy protocols. Replication jobs with proxy urls not in
; this list will fail to run.
;valid_proxy_protocols = http,https,socks5
diff --git a/src/couch_replicator/src/couch_replicator.erl b/src/couch_replicator/src/couch_replicator.erl
index 935daaa80..34c745c5d 100644
--- a/src/couch_replicator/src/couch_replicator.erl
+++ b/src/couch_replicator/src/couch_replicator.erl
@@ -58,6 +58,7 @@
replicate(PostBody, Ctx) ->
{ok, Rep0} = couch_replicator_parse:parse_rep_doc(PostBody, Ctx),
Rep = Rep0#rep{start_time = os:timestamp()},
+ ok = couch_replicator_utils:valid_endpoint_protocols_log(Rep),
#rep{id = RepId, options = Options, user_ctx = UserCtx} = Rep,
case get_value(cancel, Options, false) of
true ->
diff --git a/src/couch_replicator/src/couch_replicator_doc_processor.erl b/src/couch_replicator/src/couch_replicator_doc_processor.erl
index eb4c02b49..2a2b2d123 100644
--- a/src/couch_replicator/src/couch_replicator_doc_processor.erl
+++ b/src/couch_replicator/src/couch_replicator_doc_processor.erl
@@ -168,6 +168,7 @@ process_updated({DbName, _DocId} = Id, JsonRepDoc) ->
% problem.
Rep0 = couch_replicator_parse:parse_rep_doc_without_id(JsonRepDoc),
Rep = Rep0#rep{db_name = DbName, start_time = os:timestamp()},
+ ok = couch_replicator_utils:valid_endpoint_protocols_log(Rep),
Filter =
case couch_replicator_filters:parse(Rep#rep.options) of
{ok, nil} ->
diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
index 38533c7f2..b211da85b 100644
--- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl
+++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
@@ -78,7 +78,8 @@
use_checkpoints = true,
checkpoint_interval = ?DEFAULT_CHECKPOINT_INTERVAL,
type = db,
- view = nil
+ view = nil,
+ certificate_checker
}).
start_link(#rep{id = Id = {BaseId, Ext}, source = Src, target = Tgt} = Rep) ->
@@ -175,6 +176,8 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx = UserCtx} = Rep) -
% unfortunately not immune to race conditions.
log_replication_start(State),
+ CertificateCheckerPid = verify_ssl_certificates_log(Rep),
+
couch_log:debug("Worker pids are: ~p", [Workers]),
doc_update_triggered(Rep),
@@ -183,6 +186,7 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx = UserCtx} = Rep) -
changes_queue = ChangesQueue,
changes_manager = ChangesManager,
changes_reader = ChangesReader,
+ certificate_checker = CertificateCheckerPid,
workers = Workers
}}.
@@ -269,6 +273,8 @@ handle_info({'EXIT', Pid, {shutdown, max_backoff}}, State) ->
{stop, {shutdown, max_backoff}, State};
handle_info({'EXIT', Pid, normal}, #rep_state{changes_reader = Pid} = State) ->
{noreply, State};
+handle_info({'EXIT', Pid, _Reason}, #rep_state{certificate_checker = Pid} = State) ->
+ {noreply, State};
handle_info({'EXIT', Pid, Reason0}, #rep_state{changes_reader = Pid} = State) ->
couch_stats:increment_counter([couch_replicator, changes_reader_deaths]),
Reason =
@@ -1148,6 +1154,14 @@ log_replication_start(#rep_state{rep_details = Rep} = RepState) ->
" worker_batch_size:~p session_id:~s",
couch_log:notice(Msg, [Id, Source, Target, From, Workers, BatchSize, Sid]).
+verify_ssl_certificates_log(#rep{} = Rep) ->
+ case config:get_boolean("replicator", "verify_ssl_certificates_log", false) of
+ true ->
+ spawn_link(couch_replicator_utils, verify_ssl_certificates_log, [Rep]);
+ false ->
+ undefined
+ end.
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl
index 0aff4e964..d790acb0d 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -27,13 +27,16 @@
get_basic_auth_creds/1,
remove_basic_auth_creds/1,
normalize_basic_auth/1,
- seq_encode/1
+ seq_encode/1,
+ valid_endpoint_protocols_log/1,
+ verify_ssl_certificates_log/1
]).
-include_lib("ibrowse/include/ibrowse.hrl").
-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include_lib("public_key/include/public_key.hrl").
-import(couch_util, [
get_value/2,
@@ -277,6 +280,109 @@ seq_encode(Seq) ->
% object. We are being maximally compatible here.
?JSON_ENCODE(Seq).
+%% Log uses of http protocol
+valid_endpoint_protocols_log(#rep{} = Rep) ->
+ VerifyEnabled = config:get_boolean("replicator", "valid_endpoint_protocols_log", false),
+ case VerifyEnabled of
+ true ->
+ ok = check_endpoint_protocols(Rep, source),
+ ok = check_endpoint_protocols(Rep, target);
+ false ->
+ ok
+ end.
+
+check_endpoint_protocols(#rep{} = Rep, Type) ->
+ Url = url_from_type(Rep, Type),
+ #url{protocol = Protocol} = ibrowse_lib:parse_url(Url),
+ case Protocol of
+ http ->
+ couch_log:warning("**security warning** replication ~s has insecure ~s at ~s", [
+ rep_principal(Rep), Type, Url
+ ]),
+ ok;
+ _Else ->
+ ok
+ end.
+
+url_from_type(#rep{} = Rep, source) ->
+ Rep#rep.source#httpdb.url;
+url_from_type(#rep{} = Rep, target) ->
+ Rep#rep.target#httpdb.url.
+
+%% log uses of https protocol where verify_peer would fail.
+verify_ssl_certificates_log(#rep{} = Rep) ->
+ ok = check_ssl_certificates(Rep, source),
+ ok = check_ssl_certificates(Rep, target).
+
+check_ssl_certificates(#rep{} = Rep, Type) ->
+ VerifyEnabled = config:get_boolean("replicator", "verify_ssl_certificates", false),
+ CACertFile = config:get("replicator", "ssl_trusted_certificates_file"),
+ if
+ VerifyEnabled ->
+ % no need for an extra check if we're doing them anyway.
+ ok;
+ CACertFile == undefined ->
+ couch_log:warning(
+ "security warnings enabled but no ssl_trusted_certificates_file configured",
+ []
+ ),
+ ok;
+ true ->
+ Url = url_from_type(Rep, Type),
+ try
+ ibrowse:send_req(Url, [], head, [], [
+ {is_ssl, true},
+ {ssl_options, [
+ {cacertfile, CACertFile},
+ {verify, verify_peer},
+ {verify_fun, check_certificate_fun(Rep, Url, Type)}
+ ]}
+ ])
+ catch
+ Class:Reason ->
+ couch_log:warning("failed to check certificate of ~s (~p:~p)", [
+ Url, Class, Reason
+ ])
+ end,
+ ok
+ end.
+
+check_certificate_fun(#rep{} = Rep, Url, Type) ->
+ Fun = fun
+ (_, {bad_cert, Reason}, UserState) ->
+ couch_log:warning(
+ "**security warning** replication ~s has bad cert in ~s for reason ~p at ~s", [
+ rep_principal(Rep), Type, Reason, Url
+ ]
+ ),
+ {valid, UserState};
+ (_, {extension, #'Extension'{critical = true} = Ext}, UserState) ->
+ couch_log:warning(
+ "**security warning** replication ~s has unsupported critical extension in ~s of id ~p at ~s",
+ [
+ rep_principal(Rep), Ext#'Extension'.extnID, Url
+ ]
+ ),
+ {valid, UserState};
+ (_, {extension, _}, UserState) ->
+ {unknown, UserState};
+ (_, valid, UserState) ->
+ {valid, UserState};
+ (_, valid_peer, UserState) ->
+ {valid, UserState}
+ end,
+ InitialState = [],
+ {Fun, InitialState}.
+
+rep_principal(#rep{db_name = DbName} = Rep) when is_binary(DbName) ->
+ io_lib:format("in database ~s, docid ~s", [
+ mem3:dbname(DbName), Rep#rep.doc_id
+ ]);
+rep_principal(#rep{user_ctx = #user_ctx{name = Name}}) when is_binary(Name) ->
+ io_lib:format("by user ~s", [Name]);
+rep_principal(#rep{}) ->
+ "by unknown principal".
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/docs/src/config/replicator.rst b/src/docs/src/config/replicator.rst
index dc89ecca3..2045b2766 100644
--- a/src/docs/src/config/replicator.rst
+++ b/src/docs/src/config/replicator.rst
@@ -166,17 +166,39 @@ Replicator Database Configuration
.. _inet: http://www.erlang.org/doc/man/inet.html#setopts-2
- .. config:option:: valid_endpoint_protocols :: Replicator endpoint protocols
+ .. config:option:: valid_endpoint_protocols :: Replicator endpoint protocols
- .. versionadded:: 3.3
+ .. versionadded:: 3.3
- Valid replication endpoint protocols. Replication jobs with endpoint
- urls not in this list will fail to run::
+ Valid replication endpoint protocols. Replication jobs with endpoint
+ urls not in this list will fail to run::
- [replicator]
- valid_endpoint_protocols = http,https
+ [replicator]
+ valid_endpoint_protocols = http,https
+
+ .. config:option:: valid_endpoint_protocols_log :: Log security issues with endpoints
+
+ .. versionadded:: 3.4
+
+ When enabled, CouchDB will log any replication that uses the insecure http
+ protocol::
- .. config:option:: valid_proxy_protocols :: Replicator proxy protocols
+ [replicator]
+ valid_endpoint_protocols_log = true
+
+ .. config:option:: verify_ssl_certificates_log :: Log security issues with endpoints
+
+ .. versionadded:: 3.4
+
+ When enabled, and if ``ssl_trusted_certificates_file`` is configured
+ but ``verify_ssl_certificates`` is not, CouchDB will check the
+ validity of the TLS certificates of all sources and targets (
+ without causing the replication to fail) and log any issues::
+
+ [replicator]
+ verify_ssl_certificates_log = true
+
+ .. config:option:: valid_proxy_protocols :: Replicator proxy protocols
.. versionadded:: 3.3
@@ -307,7 +329,7 @@ Replicator Database Configuration
.. config:option:: priority_coeff :: Priority coefficient decays
- .. versionadded:: 3.2.0
+ .. versionadded:: 3.2.0
Priority coefficient decays all the job priorities such that they slowly
drift towards the front of the run queue. This coefficient defines a maximum