You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wi...@apache.org on 2026/04/27 14:39:31 UTC
(couchdb) branch wh/connect_to updated (82df75f17 -> eb3a970f1)
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a change to branch wh/connect_to
in repository https://gitbox.apache.org/repos/asf/couchdb.git
discard 82df75f17 feat: Add replicator DNS override support for outbound requests.
new eb3a970f1 feat: Add replicator DNS override support for outbound requests.
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (82df75f17)
\
N -- N -- N refs/heads/wh/connect_to (eb3a970f1)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
(couchdb) 01/01: feat: Add replicator DNS override support for outbound requests.
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch wh/connect_to
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit eb3a970f183b6ebb78852b4212079f569de82384
Author: Will Holley <wi...@uk.ibm.com>
AuthorDate: Mon Apr 27 13:53:28 2026 +0100
feat: Add replicator DNS override support for outbound requests.
This adds a feature to the CouchDB replicator
to override the DNS target for specific host
patterns (including wildcards) when making
outbound requests. The use case is when requests need
to be routed via a transparent SNI proxy e.g.
for network egress monitoring.
There is adds a new configuration option to
specify the overrides:
```
[replicator]
dns_overrides = host:target, host2:target
```
The replicator resolves the configured host patterns
to the alternative connection targets while
preserving the request URL host (applies to
regular requests and session-auth requests).
Note this depends on the `connect_to` option in
ibrowse, which is a custom feature in the CouchDB
ibrowse fork.
---
rel/overlay/etc/default.ini | 3 +
.../src/couch_replicator_auth_session.erl | 27 ++++++-
src/couch_replicator/src/couch_replicator_dns.erl | 92 ++++++++++++++++++++++
.../src/couch_replicator_httpc.erl | 33 +++++++-
.../test/eunit/couch_replicator_dns_tests.erl | 63 +++++++++++++++
5 files changed, 216 insertions(+), 2 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index bb017fb44..ff22bcf4f 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -707,6 +707,9 @@ partitioned||* = true
; Checkpoint interval
;checkpoint_interval = 30000
+; DNS overrides for replication requests
+;dns_overrides = *.example.test:127.0.0.1
+
; Some socket options that might boost performance in some scenarios:
; {nodelay, boolean()}
; {sndbuf, integer()}
diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl b/src/couch_replicator/src/couch_replicator_auth_session.erl
index 93ddc834f..c66086926 100644
--- a/src/couch_replicator/src/couch_replicator_auth_session.erl
+++ b/src/couch_replicator/src/couch_replicator_auth_session.erl
@@ -311,11 +311,36 @@ refresh(#state{session_url = Url, user = User, pass = Pass} = State) ->
{ok, string(), headers(), binary()} | {error, term()}.
http_request(#state{httpdb_pool = Pool} = State, Url, Headers, Method, Body) ->
Timeout = State#state.httpdb_timeout,
- Opts = [
+
+ % Apply DNS override using connect_to ibrowse option
+ ParsedUrl = ibrowse_lib:parse_url(Url),
+ Host = element(3, ParsedUrl), % #url.host
+ Proto = element(2, ParsedUrl), % #url.protocol
+ {TargetHost, OriginalHost} = couch_replicator_dns:resolve_host(Host),
+
+ Opts0 = [
{response_format, binary},
{inactivity_timeout, Timeout}
| State#state.httpdb_ibrowse_options
],
+
+ % Add connect_to ibrowse option if DNS override is active
+ Opts1 = case OriginalHost of
+ undefined ->
+ Opts0;
+ _ ->
+ couch_log:debug("DNS override for session: ~s -> ~s", [OriginalHost, TargetHost]),
+ [{connect_to, TargetHost} | Opts0]
+ end,
+
+ % Add SNI for HTTPS with DNS override
+ Opts = case {Proto, OriginalHost} of
+ {https, OrigHost} when is_list(OrigHost) ->
+ couch_replicator_httpc:add_sni_option(Opts1, OrigHost);
+ _ ->
+ Opts1
+ end,
+
{ok, Wrk} = couch_replicator_httpc_pool:get_worker(Pool),
try
Result = ibrowse:send_req_direct(
diff --git a/src/couch_replicator/src/couch_replicator_dns.erl b/src/couch_replicator/src/couch_replicator_dns.erl
new file mode 100644
index 000000000..985f2c25a
--- /dev/null
+++ b/src/couch_replicator/src/couch_replicator_dns.erl
@@ -0,0 +1,92 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_dns).
+
+-export([
+ resolve_host/1,
+ parse_config/1,
+ match_pattern/2,
+ get_overrides/0
+]).
+
+-type dns_override() :: {binary(), binary()}.
+
+-spec resolve_host(string()) -> {string(), string() | undefined}.
+resolve_host(Host) ->
+ case find_override(unicode:characters_to_binary(Host), get_overrides()) of
+ {ok, Target} ->
+ {binary_to_list(Target), Host};
+ not_found ->
+ {Host, undefined}
+ end.
+
+-spec get_overrides() -> [dns_override()].
+get_overrides() ->
+ case config:get("replicator", "dns_overrides", undefined) of
+ undefined ->
+ [];
+ ConfigStr ->
+ parse_config(ConfigStr)
+ end.
+
+-spec parse_config(string()) -> [dns_override()].
+parse_config(ConfigStr) ->
+ Entries = binary:split(
+ unicode:characters_to_binary(ConfigStr), <<",">>, [global, trim]
+ ),
+ lists:filtermap(fun parse_entry/1, Entries).
+
+parse_entry(<<>>) ->
+ false;
+parse_entry(Entry0) ->
+ Entry = string:trim(Entry0),
+ case binary:split(Entry, <<":">>) of
+ [Pattern0, Target0] ->
+ Pattern = string:trim(Pattern0),
+ Target = string:trim(Target0),
+ case {Pattern, Target} of
+ {<<>>, _} -> invalid_entry(Entry);
+ {_, <<>>} -> invalid_entry(Entry);
+ _ -> {true, {Pattern, Target}}
+ end;
+ _ ->
+ invalid_entry(Entry)
+ end.
+
+invalid_entry(Entry) ->
+ couch_log:warning("Invalid dns_override entry: ~ts", [Entry]),
+ false.
+
+find_override(_Host, []) ->
+ not_found;
+find_override(Host, [{Pattern, Target} | Rest]) ->
+ case match_pattern(Host, Pattern) of
+ true ->
+ {ok, Target};
+ false ->
+ find_override(Host, Rest)
+ end.
+
+-spec match_pattern(binary() | string(), binary() | string()) -> boolean().
+match_pattern(Host0, Pattern0) ->
+ Host = unicode:characters_to_binary(Host0),
+ Pattern = unicode:characters_to_binary(Pattern0),
+ match_pattern_binary(Host, Pattern).
+
+match_pattern_binary(Host, <<"*", Suffix/binary>>) ->
+ HostSize = byte_size(Host),
+ SuffixSize = byte_size(Suffix),
+ HostSize >= SuffixSize andalso
+ binary:part(Host, HostSize - SuffixSize, SuffixSize) =:= Suffix;
+match_pattern_binary(Host, Pattern) ->
+ Host =:= Pattern.
\ No newline at end of file
diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl
index fe81e65ea..f99c8edea 100644
--- a/src/couch_replicator/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator/src/couch_replicator_httpc.erl
@@ -20,6 +20,7 @@
-export([send_req/3]).
-export([stop_http_worker/0]).
-export([full_url/2]).
+-export([add_sni_option/2]).
-import(couch_util, [
get_value/2,
@@ -113,6 +114,10 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) ->
{Headers2, HttpDb} = couch_replicator_auth:update_headers(HttpDb0, Headers1),
Url = full_url(HttpDb, Params),
Body = get_value(body, Params, []),
+
+ % Apply DNS override using connect_to ibrowse option
+ #url{host = Host, protocol = Protocol} = ibrowse_lib:parse_url(Url),
+ {TargetHost, OriginalHost} = couch_replicator_dns:resolve_host(Host),
case get_value(path, Params) == "_changes" of
true ->
Timeout = infinity;
@@ -131,7 +136,7 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) ->
{User, Pass} when is_list(User), is_list(Pass) ->
[{basic_auth, {User, Pass}}]
end,
- IbrowseOptions =
+ IbrowseOptions0 =
BasicAuthOpts ++
[
{response_format, binary},
@@ -142,6 +147,25 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) ->
HttpDb#httpdb.ibrowse_options
)
],
+
+ % Add connect_to ibrowse option if DNS override is active
+ IbrowseOptions1 = case OriginalHost of
+ undefined ->
+ IbrowseOptions0;
+ _ ->
+ % Log DNS override for debugging
+ couch_log:debug("DNS override: ~s -> ~s", [OriginalHost, TargetHost]),
+ [{connect_to, TargetHost} | IbrowseOptions0]
+ end,
+
+ % Add SNI for HTTPS with DNS override
+ IbrowseOptions = case {Protocol, OriginalHost} of
+ {https, OrigHost} when is_list(OrigHost) ->
+ add_sni_option(IbrowseOptions1, OrigHost);
+ _ ->
+ IbrowseOptions1
+ end,
+
backoff_before_request(Worker, HttpDb, Params),
Response = ibrowse:send_req_direct(
Worker, Url, Headers2, Method, Body, IbrowseOptions, Timeout
@@ -535,6 +559,13 @@ merge_headers(Headers1, Headers2) when is_list(Headers1), is_list(Headers2) ->
Merged = mochiweb_headers:enter_from_list(Headers1 ++ Headers2, Empty),
mochiweb_headers:to_list(Merged).
+%% @private Add SNI to SSL options
+add_sni_option(IbrowseOpts, Host) ->
+ SslOpts = proplists:get_value(ssl_options, IbrowseOpts, []),
+ SslOpts1 = [{server_name_indication, Host} |
+ proplists:delete(server_name_indication, SslOpts)],
+ lists:keystore(ssl_options, 1, IbrowseOpts, {ssl_options, SslOpts1}).
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/couch_replicator/test/eunit/couch_replicator_dns_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_dns_tests.erl
new file mode 100644
index 000000000..c260008d0
--- /dev/null
+++ b/src/couch_replicator/test/eunit/couch_replicator_dns_tests.erl
@@ -0,0 +1,63 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_dns_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+
+match_pattern_test_() ->
+ [
+ ?_assert(couch_replicator_dns:match_pattern(
+ "account.example.test", "*.example.test")),
+ ?_assertNot(couch_replicator_dns:match_pattern(
+ "example.test", "*.example.test")),
+ ?_assert(couch_replicator_dns:match_pattern(
+ "exact.example.test", "exact.example.test")),
+ ?_assertNot(couch_replicator_dns:match_pattern(
+ "other.example.test", "exact.example.test"))
+ ].
+
+parse_config_test_() ->
+ [
+ ?_assertEqual(
+ 2,
+ length(couch_replicator_dns:parse_config(
+ "*.example.test:proxy.internal, exact.example.test:127.0.0.1"
+ ))
+ ),
+ ?_assertEqual([], couch_replicator_dns:parse_config(""))
+ ].
+
+resolve_host_test_() ->
+ {setup,
+ fun() ->
+ meck:new(config, [passthrough]),
+ meck:expect(config, get, fun
+ ("replicator", "dns_overrides", _) ->
+ "*.example.test:egress.internal";
+ (_, _, Default) ->
+ Default
+ end)
+ end,
+ fun(_) ->
+ meck:unload(config)
+ end,
+ [
+ ?_assertEqual(
+ {"egress.internal", "account.example.test"},
+ couch_replicator_dns:resolve_host("account.example.test")
+ ),
+ ?_assertEqual(
+ {"other.example.org", undefined},
+ couch_replicator_dns:resolve_host("other.example.org")
+ )
+ ]}.