You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@couchdb.apache.org by "rnewson (via GitHub)" <gi...@apache.org> on 2026/07/17 12:25:26 UTC
[PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
rnewson opened a new pull request, #6069:
URL: https://github.com/apache/couchdb/pull/6069
## Overview
Teach the CouchDB replicator how to acquire an access token from an apikey from IBM's IAM authentication service.
Note that an access token has an expiration (currently one hour but subject to change).
The plugin fetches a new token 5 minutes before the currently held one expires.
## Testing recommendations
will be covered by tests
## Related Issues or Pull Requests
N/A
## Checklist
- [x] This is my own work, I did not use AI, LLM's or similar technology
- [x] Code is written and works correctly
- [ ] Changes are covered by tests
- [x] Any new configurable parameters are documented in `rel/overlay/etc/default.ini`
- [x] Documentation changes were made in the `src/docs` folder
- [ ] Documentation changes were backported (separated PR) to affected branches
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3604156649
##########
src/couch_replicator/src/couch_replicator_auth_ibm_iam.erl:
##########
@@ -0,0 +1,178 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_auth_ibm_iam).
+
+-behaviour(couch_replicator_auth).
+
+-export([
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+-export([token_acquirer/1]).
+
+-define(EARLY_REFRESH_MS, 5 * 60 * 1000).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include_lib("ibrowse/include/ibrowse.hrl").
+
+-record(state, {
+ acquirer,
+ apikey :: binary(),
+ refresh_at_ms,
+ token :: list()
+}).
+
+%% callbacks
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_apikey(HttpDb) of
+ ignore ->
+ ignore;
+ {ok, APIKey} ->
+ Acquirer = spawn_link(?MODULE, token_acquirer, [APIKey]),
Review Comment:
It might be better to do a first time synchronous token fetch on init. So when the job starts we'd know right away it has an invalid creds
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r4132730423
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,719 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ last_used,
+ last_updated,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ gen_server:cast(?MODULE, {update_last_used, APIKeyMAC}),
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ last_updated = NowSecs
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ ok;
+ true ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.last_used >= Entry#private_entry.last_updated of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ AcquireMRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{acquire_mref = AcquireMRef});
+ false ->
+ %% let it expire and free any waiters
+ reply_and_reset(Entry, {error, token_expiring})
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ NowSecs = now_secs(),
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{expires_at_secs = ExpiresAtSecs} = Entry] when ExpiresAtSecs =< NowSecs ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ demonitor(Entry#private_entry.acquire_mref, [flush]);
+ true ->
+ ok
+ end,
+ reply_all(Entry#private_entry.waiters, {error, expired_api_key_entry});
+ % was refreshed
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(refresh_expire_check, #state{} = State) ->
+ NowSecs = now_secs(),
+ EarlyRefreshSecs = early_refresh_secs(),
+ Matches = ets:select(?PRIVATE, [
+ {
+ #private_entry{api_key_mac = '$1', expires_at_secs = '$2', _ = '_'},
+ [{is_integer, '$2'}, {'=<', '$2', NowSecs + EarlyRefreshSecs}],
+ [['$1', '$2']]
+ }
+ ]),
+ lists:foreach(
+ fun([APIKeyMAC, ExpiresAtSecs]) ->
+ if
+ NowSecs >= ExpiresAtSecs ->
+ self() ! {expire_api_key_entry, APIKeyMAC};
+ true ->
+ erlang:send_after(jitter_ms(), self(), {refresh_token, APIKeyMAC})
+ end
+ end,
+ Matches
+ ),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {noreply, State};
+handle_info(
+ {'DOWN', AcquireMRef, process, _AcquirePid, {ok, StatusCode, _ResponseHeaders, ResponseBody}},
+ #state{} = State
+) ->
+ case match_on_acquire_mref(AcquireMRef) of
+ [#private_entry{} = Entry] ->
+ case StatusCode of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresAtSecs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ true = ets:update_element(?PUBLIC, Entry#private_entry.api_key_mac, {
+ #public_entry.token, Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ expires_at_secs = ExpiresAtSecs,
+ last_updated = now_secs()
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ retry_ms(),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({'DOWN', AcquireMRef, process, _AcquirePid, Reason}, #state{} = State) ->
+ %% acquire process exited with {error, Reason} or died some other way
+ %% we don't care how, we always clean up after it
+ case match_on_acquire_mref(AcquireMRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, acquire_error(Reason));
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
+handle_info(token_url_change, State) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ %% picked up by the next request, nothing to reconnect
+ {noreply, State#state{token_url = TokenUrl}};
+ {error, Reason} ->
+ {stop, Reason, State}
+ end;
+handle_info(Msg, State) ->
+ couch_log:warning("~p: unexpected info ~p", [?MODULE, Msg]),
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ets:foldl(
+ fun(#private_entry{} = Entry, Acc) ->
+ lists:foreach(
+ fun(W) -> gen_server:reply(W, {error, terminated}) end,
+ Entry#private_entry.waiters
+ ),
+ Acc
+ end,
+ ok,
+ ?PRIVATE
+ ).
+
+%% config_listener functions.
+
+handle_config_change("replicator_auth_ibm", "token_url", _, _, State) ->
+ ?MODULE ! token_url_change,
+ {ok, State};
+handle_config_change(_, _, _, _, State) ->
+ {ok, State}.
+
+handle_config_terminate(_, stop, _) ->
+ ok;
+handle_config_terminate(_Server, _Reason, _State) ->
+ erlang:send_after(5000, ?MODULE, restart_config_listener).
+
+%% private functions.
+
+now_secs() ->
+ erlang:system_time(second).
+
+extract_api_key(#httpdb{auth_props = AuthProps}) ->
+ case proplists:get_value(~"ibm", AuthProps) of
+ {IBMProps} when is_list(IBMProps) ->
+ case proplists:get_value(~"apikey", IBMProps) of
+ APIKey when is_binary(APIKey), byte_size(APIKey) > 0 ->
+ {ok, APIKey};
+ _ ->
+ {error, missing_api_key}
+ end;
+ _ ->
+ {error, missing_api_key}
+ end.
+
+acquire_token(APIKey, #state{token_url = Url}) ->
+ Headers = [
+ {~"content-type", ~"application/x-www-form-urlencoded"}
+ ],
+ Body = mochiweb_util:urlencode([
+ {~"grant_type", ~"urn:ibm:params:oauth:grant-type:apikey"},
+ {~"response_type", ~"cloud_iam"},
+ {~"apikey", APIKey}
+ ]),
+ {_Pid, Ref} = spawn_monitor(fun() ->
+ exit(
+ try
+ post(Url, Headers, Body, request_timeout_ms())
+ catch
+ Tag:Err -> {error, {Tag, Err}}
+ end
+ )
+ end),
+ Ref.
+
+post(Url, ReqHeaders, ReqBody, Timeout) ->
+ Opts = #{
+ timeout => Timeout,
+ tls_opts => [{cacerts, couch_replicator_utils:cacert_get()}]
+ },
+ couch_gun:req(post, Url, ReqHeaders, ReqBody, Opts).
+
+decode_iam_response(ResponseBody) ->
+ try jiffy:decode(ResponseBody, [return_maps]) of
+ Decoded when is_map(Decoded) ->
+ Token = maps:get(~"access_token", Decoded, undefined),
+ ExpiresAtSecs = maps:get(~"expiration", Decoded, undefined),
+ if
+ is_binary(Token) andalso is_integer(ExpiresAtSecs) ->
+ {ok, Token, ExpiresAtSecs};
+ true ->
+ {error, malformed_iam_response}
+ end;
+ _ ->
+ {error, malformed_iam_response}
+ catch
+ _:_ ->
+ {error, malformed_iam_response}
+ end.
+
+extract_error_message(ResponseBody) ->
+ try jiffy:decode(ResponseBody, [return_maps]) of
+ #{~"errorCode" := ErrorCode, ~"errorMessage" := ErrorMessage} ->
+ {ErrorCode, ErrorMessage};
+ _ ->
+ malformed_iam_response
+ catch
+ _:_ ->
+ malformed_iam_response
+ end.
+
+api_key_uuid(EncodedToken) when is_binary(EncodedToken) ->
+ case binary:split(EncodedToken, ~".", [global]) of
+ [_Header, Payload, _Signature] ->
+ case b64url:decode(Payload) of
+ {error, _Reason} ->
+ ~"";
+ Decoded when is_binary(Decoded) ->
+ try jiffy:decode(Decoded, [return_maps]) of
+ #{~"apikey_uuid" := UUID} ->
+ UUID;
+ _ ->
+ ~""
+ catch
+ _:_ ->
+ ~""
+ end
+ end;
+ _ ->
+ ~""
+ end.
+
+-spec token_url() -> {ok, string()} | {error, term()}.
+token_url() ->
+ Url = config:get(
+ "replicator_auth_ibm", "token_url", "https://iam.cloud.ibm.com/identity/token"
+ ),
+ case couch_gun:parse_url(Url) of
+ {ok, #{}} ->
+ {ok, Url};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+match_on_acquire_mref(AcquireMRef) ->
+ ets:match_object(?PRIVATE, #private_entry{acquire_mref = AcquireMRef, _ = '_'}).
+
+acquire_error({error, _} = Error) ->
+ Error;
+acquire_error(Reason) ->
+ {error, Reason}.
+
+-spec reply_and_reset(Entry :: #private_entry{}, Reply :: term()) -> ok.
+reply_and_reset(#private_entry{} = Entry, Reply) ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ acquire_mref = undefined,
+ waiters = []
+ }),
+ reply_all(Entry#private_entry.waiters, Reply).
+
+-spec reply_all(Waiters :: [gen_server:from()], Reply :: term()) -> ok.
+reply_all(Waiters, Reply) ->
+ lists:foreach(
+ fun(W) -> gen_server:reply(W, Reply) end,
+ Waiters
+ ).
+
+mac(Key, Data) when is_binary(Key), is_binary(Data) ->
+ crypto:mac(hmac, sha256, Key, Data).
+
+request_timeout_ms() ->
+ max(0, 1000 * config:get_integer("replicator_auth_ibm", "request_timeout_secs", 30)).
+
+check_interval_ms() ->
+ max(0, 1000 * config:get_integer("replicator_auth_ibm", "check_interval_secs", 60)).
+
+early_refresh_secs() ->
+ max(0, config:get_integer("replicator_auth_ibm", "early_refresh_secs", 300)).
+
+jitter_ms() ->
+ JitterMS = max(0, 1000 * config:get_integer("replicator_auth_ibm", "jitter_secs", 60)),
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3913460399
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ false ->
+ %% let it expire
+ ok
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, fin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, {unexpected_status_code, StatusCode}});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+
+ RefreshRef = erlang:send_after(
+ max(
+ ?MIN_REFRESH_MS,
+ ExpiresInMs - ?EARLY_REFRESH_MS - rand:uniform(?JITTER_MS)
+ ),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ expires_ref = ExpiresRef,
+ refresh_ref = RefreshRef,
+ token_updated_at = now()
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ ?MIN_REFRESH_MS,
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_error, GunPid, GunStreamRef, Reason}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_error, GunPid, Reason}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun error ~p", [?MODULE, Reason]),
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
+handle_info({gun_up, GunPid, _Protocol}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, closed, []}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, Reason, KilledStreams}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun connection down for reason: ~p", [?MODULE, Reason]),
+ lists:foreach(
+ fun(GunStreamRef) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ [] ->
+ ok
+ end
+ end,
+ KilledStreams
+ ),
+ {noreply, State};
+handle_info(restart_gun, State) ->
+ case start_gun(State) of
+ {ok, NewState} ->
+ {noreply, NewState};
+ {error, Reason} ->
+ couch_log:warning("~p: gun restart failed for reason: ~p", [?MODULE, Reason]),
+ erlang:send_after(5000, self(), restart_gun),
+ {noreply, State}
+ end;
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
+handle_info(token_url_change, State0) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ stop_gun(State0),
Review Comment:
stop_gun now demonitors and flushes
##########
src/docs/src/replication/replicator.rst:
##########
@@ -809,6 +809,28 @@ they are used. If they are not, then URL userinfo is checked. If credentials
are found there, then those credentials are used, otherwise basic auth header
is used.
+Using an IBM IAM apikey
+=======================
+
+If you've enabled the optional `couch_replicator_auth_ibm_auth`
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3980402902
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,697 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ refresh_ref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
Review Comment:
A successful acquire will clear it. The only time it would be false is across a failed acquire
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3978897870
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,697 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ refresh_ref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
Review Comment:
ah should be `used_since_acquisition` now, good catch. It makes sense to reset the expiration if a new replication has just started that happens to use an apikey we already have registered.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3786050658
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
Review Comment:
Is there a gun_response with `fin` to handle? Got headers but no body, maybe some redirect or 500 case?
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
Review Comment:
In some places dealing with periodic execution (replication jobs starts) we usually add a bit of jitter to avoid a thundering herd problem wonder if we should add a bit of that here as well.
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ refresh_ref = RefreshRef,
+ expires_ref = ExpiresRef
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ ?MIN_REFRESH_MS,
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
+handle_info({gun_up, GunPid, _Protocol}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, closed, []}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, Reason, KilledStreams}, #state{gun_pid = GunPid} = State) ->
Review Comment:
Gun docs indicated there are also `gun_error` messages send and we don't have clauses for them
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ refresh_ref = RefreshRef,
+ expires_ref = ExpiresRef
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ ?MIN_REFRESH_MS,
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
+handle_info({gun_up, GunPid, _Protocol}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, closed, []}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, Reason, KilledStreams}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun connection down for reason: ~p", [?MODULE, Reason]),
+ lists:foreach(
+ fun(GunStreamRef) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ [] ->
+ ok
+ end
+ end,
+ KilledStreams
+ ),
+ {noreply, State};
+handle_info(restart_gun, State) ->
+ case start_gun(State) of
+ {ok, NewState} ->
+ {noreply, NewState};
+ {error, Reason} ->
+ couch_log:warning("~p: gun restart failed for reason: ~p", [?MODULE, Reason]),
+ erlang:send_after(5000, self(), restart_gun),
+ {noreply, State}
+ end;
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
+handle_info(token_url_change, State) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ {noreply, State#state{token_uri_map = TokenURIMap}};
Review Comment:
Wonder if we should clear connections or reset anything with the old url here
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
Review Comment:
If we don't do cleanup does it mean we'd end up refreshing a once used API key forever even if there are no more jobs using it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3978932096
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,697 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ refresh_ref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
Review Comment:
I want it to expire if not used after registration (or after a subsequent refresh).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#issuecomment-5006989234
I do want to think of a more generic thing, and we should maybe make it possible for an auth plugin to have application like semantics (so they can start a supervision tree etc). in that version things become simpler. the auth plugin would launch a gen_server that makes a public ETS table. each replication job would insert the apikey when it starts and then fetch the token in update_headers as needed. the gen_server could refresh all the tokens slightly in advance of their expiration, handle retries on timeout or 500's, etc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3787159264
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ refresh_ref = RefreshRef,
+ expires_ref = ExpiresRef
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ ?MIN_REFRESH_MS,
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
+handle_info({gun_up, GunPid, _Protocol}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, closed, []}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, Reason, KilledStreams}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun connection down for reason: ~p", [?MODULE, Reason]),
+ lists:foreach(
+ fun(GunStreamRef) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ [] ->
+ ok
+ end
+ end,
+ KilledStreams
+ ),
+ {noreply, State};
+handle_info(restart_gun, State) ->
+ case start_gun(State) of
+ {ok, NewState} ->
+ {noreply, NewState};
+ {error, Reason} ->
+ couch_log:warning("~p: gun restart failed for reason: ~p", [?MODULE, Reason]),
+ erlang:send_after(5000, self(), restart_gun),
+ {noreply, State}
+ end;
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
+handle_info(token_url_change, State) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ {noreply, State#state{token_uri_map = TokenURIMap}};
Review Comment:
done
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ refresh_ref = RefreshRef,
+ expires_ref = ExpiresRef
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ ?MIN_REFRESH_MS,
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
+handle_info({gun_up, GunPid, _Protocol}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, closed, []}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, Reason, KilledStreams}, #state{gun_pid = GunPid} = State) ->
Review Comment:
done
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3786997632
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ RefreshRef = erlang:send_after(
+ max(?MIN_REFRESH_MS, ExpiresInMs - ?EARLY_REFRESH_MS),
Review Comment:
I figure they'd be staggered already but can certainly add some jitter here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r4126011653
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,719 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ last_used,
+ last_updated,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ gen_server:cast(?MODULE, {update_last_used, APIKeyMAC}),
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ last_updated = NowSecs
Review Comment:
These get the same second and first acquire can hit the same second as registration then an unused key >= will be true we'll refresh. Maybe last_updated should = 0 here when we register and then we can compare with `>` in `Entry#private_entry.last_used > Entry#private_entry.last_updated`.
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,719 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ last_used,
+ last_updated,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ gen_server:cast(?MODULE, {update_last_used, APIKeyMAC}),
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ last_updated = NowSecs
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ ok;
+ true ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.last_used >= Entry#private_entry.last_updated of
Review Comment:
See note about hitting the same second here. `last_updated = 0` on registration and `>` here should fix I think...
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,719 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ last_used,
+ last_updated,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ gen_server:cast(?MODULE, {update_last_used, APIKeyMAC}),
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ last_updated = NowSecs
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ ok;
+ true ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.last_used >= Entry#private_entry.last_updated of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ AcquireMRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{acquire_mref = AcquireMRef});
+ false ->
+ %% let it expire and free any waiters
+ reply_and_reset(Entry, {error, token_expiring})
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ NowSecs = now_secs(),
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{expires_at_secs = ExpiresAtSecs} = Entry] when ExpiresAtSecs =< NowSecs ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ demonitor(Entry#private_entry.acquire_mref, [flush]);
+ true ->
+ ok
+ end,
+ reply_all(Entry#private_entry.waiters, {error, expired_api_key_entry});
+ % was refreshed
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(refresh_expire_check, #state{} = State) ->
+ NowSecs = now_secs(),
+ EarlyRefreshSecs = early_refresh_secs(),
+ Matches = ets:select(?PRIVATE, [
+ {
+ #private_entry{api_key_mac = '$1', expires_at_secs = '$2', _ = '_'},
+ [{is_integer, '$2'}, {'=<', '$2', NowSecs + EarlyRefreshSecs}],
+ [['$1', '$2']]
+ }
+ ]),
+ lists:foreach(
+ fun([APIKeyMAC, ExpiresAtSecs]) ->
+ if
+ NowSecs >= ExpiresAtSecs ->
+ self() ! {expire_api_key_entry, APIKeyMAC};
+ true ->
+ erlang:send_after(jitter_ms(), self(), {refresh_token, APIKeyMAC})
+ end
+ end,
+ Matches
+ ),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {noreply, State};
+handle_info(
+ {'DOWN', AcquireMRef, process, _AcquirePid, {ok, StatusCode, _ResponseHeaders, ResponseBody}},
+ #state{} = State
+) ->
+ case match_on_acquire_mref(AcquireMRef) of
+ [#private_entry{} = Entry] ->
+ case StatusCode of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresAtSecs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ true = ets:update_element(?PUBLIC, Entry#private_entry.api_key_mac, {
+ #public_entry.token, Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ expires_at_secs = ExpiresAtSecs,
+ last_updated = now_secs()
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ retry_ms(),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({'DOWN', AcquireMRef, process, _AcquirePid, Reason}, #state{} = State) ->
+ %% acquire process exited with {error, Reason} or died some other way
+ %% we don't care how, we always clean up after it
+ case match_on_acquire_mref(AcquireMRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, acquire_error(Reason));
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
+handle_info(token_url_change, State) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ %% picked up by the next request, nothing to reconnect
+ {noreply, State#state{token_url = TokenUrl}};
+ {error, Reason} ->
+ {stop, Reason, State}
+ end;
+handle_info(Msg, State) ->
+ couch_log:warning("~p: unexpected info ~p", [?MODULE, Msg]),
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ets:foldl(
+ fun(#private_entry{} = Entry, Acc) ->
+ lists:foreach(
+ fun(W) -> gen_server:reply(W, {error, terminated}) end,
+ Entry#private_entry.waiters
+ ),
+ Acc
+ end,
+ ok,
+ ?PRIVATE
+ ).
+
+%% config_listener functions.
+
+handle_config_change("replicator_auth_ibm", "token_url", _, _, State) ->
+ ?MODULE ! token_url_change,
+ {ok, State};
+handle_config_change(_, _, _, _, State) ->
+ {ok, State}.
+
+handle_config_terminate(_, stop, _) ->
+ ok;
+handle_config_terminate(_Server, _Reason, _State) ->
+ erlang:send_after(5000, ?MODULE, restart_config_listener).
+
+%% private functions.
+
+now_secs() ->
+ erlang:system_time(second).
+
+extract_api_key(#httpdb{auth_props = AuthProps}) ->
+ case proplists:get_value(~"ibm", AuthProps) of
+ {IBMProps} when is_list(IBMProps) ->
+ case proplists:get_value(~"apikey", IBMProps) of
+ APIKey when is_binary(APIKey), byte_size(APIKey) > 0 ->
+ {ok, APIKey};
+ _ ->
+ {error, missing_api_key}
+ end;
+ _ ->
+ {error, missing_api_key}
+ end.
+
+acquire_token(APIKey, #state{token_url = Url}) ->
+ Headers = [
+ {~"content-type", ~"application/x-www-form-urlencoded"}
+ ],
+ Body = mochiweb_util:urlencode([
+ {~"grant_type", ~"urn:ibm:params:oauth:grant-type:apikey"},
+ {~"response_type", ~"cloud_iam"},
+ {~"apikey", APIKey}
+ ]),
+ {_Pid, Ref} = spawn_monitor(fun() ->
+ exit(
+ try
+ post(Url, Headers, Body, request_timeout_ms())
+ catch
+ Tag:Err -> {error, {Tag, Err}}
+ end
+ )
+ end),
+ Ref.
+
+post(Url, ReqHeaders, ReqBody, Timeout) ->
+ Opts = #{
+ timeout => Timeout,
+ tls_opts => [{cacerts, couch_replicator_utils:cacert_get()}]
+ },
+ couch_gun:req(post, Url, ReqHeaders, ReqBody, Opts).
+
+decode_iam_response(ResponseBody) ->
+ try jiffy:decode(ResponseBody, [return_maps]) of
+ Decoded when is_map(Decoded) ->
+ Token = maps:get(~"access_token", Decoded, undefined),
+ ExpiresAtSecs = maps:get(~"expiration", Decoded, undefined),
+ if
+ is_binary(Token) andalso is_integer(ExpiresAtSecs) ->
+ {ok, Token, ExpiresAtSecs};
+ true ->
+ {error, malformed_iam_response}
+ end;
+ _ ->
+ {error, malformed_iam_response}
+ catch
+ _:_ ->
+ {error, malformed_iam_response}
+ end.
+
+extract_error_message(ResponseBody) ->
+ try jiffy:decode(ResponseBody, [return_maps]) of
+ #{~"errorCode" := ErrorCode, ~"errorMessage" := ErrorMessage} ->
+ {ErrorCode, ErrorMessage};
+ _ ->
+ malformed_iam_response
+ catch
+ _:_ ->
+ malformed_iam_response
+ end.
+
+api_key_uuid(EncodedToken) when is_binary(EncodedToken) ->
+ case binary:split(EncodedToken, ~".", [global]) of
+ [_Header, Payload, _Signature] ->
+ case b64url:decode(Payload) of
+ {error, _Reason} ->
+ ~"";
+ Decoded when is_binary(Decoded) ->
+ try jiffy:decode(Decoded, [return_maps]) of
+ #{~"apikey_uuid" := UUID} ->
+ UUID;
+ _ ->
+ ~""
+ catch
+ _:_ ->
+ ~""
+ end
+ end;
+ _ ->
+ ~""
+ end.
+
+-spec token_url() -> {ok, string()} | {error, term()}.
+token_url() ->
+ Url = config:get(
+ "replicator_auth_ibm", "token_url", "https://iam.cloud.ibm.com/identity/token"
+ ),
+ case couch_gun:parse_url(Url) of
+ {ok, #{}} ->
+ {ok, Url};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+match_on_acquire_mref(AcquireMRef) ->
+ ets:match_object(?PRIVATE, #private_entry{acquire_mref = AcquireMRef, _ = '_'}).
+
+acquire_error({error, _} = Error) ->
+ Error;
+acquire_error(Reason) ->
+ {error, Reason}.
+
+-spec reply_and_reset(Entry :: #private_entry{}, Reply :: term()) -> ok.
+reply_and_reset(#private_entry{} = Entry, Reply) ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ acquire_mref = undefined,
+ waiters = []
+ }),
+ reply_all(Entry#private_entry.waiters, Reply).
+
+-spec reply_all(Waiters :: [gen_server:from()], Reply :: term()) -> ok.
+reply_all(Waiters, Reply) ->
+ lists:foreach(
+ fun(W) -> gen_server:reply(W, Reply) end,
+ Waiters
+ ).
+
+mac(Key, Data) when is_binary(Key), is_binary(Data) ->
+ crypto:mac(hmac, sha256, Key, Data).
+
+request_timeout_ms() ->
+ max(0, 1000 * config:get_integer("replicator_auth_ibm", "request_timeout_secs", 30)).
+
+check_interval_ms() ->
+ max(0, 1000 * config:get_integer("replicator_auth_ibm", "check_interval_secs", 60)).
+
+early_refresh_secs() ->
+ max(0, config:get_integer("replicator_auth_ibm", "early_refresh_secs", 300)).
+
+jitter_ms() ->
+ JitterMS = max(0, 1000 * config:get_integer("replicator_auth_ibm", "jitter_secs", 60)),
Review Comment:
There is a discrepancy with the default.ini value of 10.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3964610783
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,697 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ refresh_ref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
Review Comment:
This should be gone now?
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,697 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ refresh_ref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{acquire_mref = AcquireMRef} when AcquireMRef /= undefined ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({used_since_acquisition, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.used_since_acquisition, true}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.used_since_acquisition of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ AcquireMRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{acquire_mref = AcquireMRef});
+ false ->
+ %% let it expire
Review Comment:
If we fail on first acquire we's still be stuck I think. A newly registered key won't be registered as used so even if IAM remote heals, we'll be stuck in a loop?
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,697 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ refresh_ref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
Review Comment:
Wonder if we should consider it `true` and only a successful acquire clears it to false
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3917422659
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,792 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now_secs()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ false ->
+ %% let it expire
+ ok
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, fin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, {unexpected_status_code, StatusCode}});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+
+ RefreshRef = erlang:send_after(
+ max(
+ min_refresh_ms(),
+ ExpiresInMs - early_refresh_ms() - rand:uniform(jitter_ms())
+ ),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ expires_ref = ExpiresRef,
+ refresh_ref = RefreshRef,
+ token_updated_at = now_secs()
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ min_refresh_ms(),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_error, GunPid, GunStreamRef, Reason}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_error, GunPid, Reason}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun error ~p", [?MODULE, Reason]),
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
Review Comment:
What happens to any in-flight stream refs in this case? If the process just goes away we'd still be left with stale stream refs referencing refreshes in process that will never finish.
We do:
```
#private_entry{gun_stream_ref = GunStreamRef} when
GunStreamRef /= undefined
->
```
But we don't match and see if it's an active ref or something that went away and if match some ref (not necessarily) the current ref we'd fall through and wait for a refresh to finish. Perhaps we have a guarantees that when gun just never crashes or dies that way without sending us something back...but without that we'd want some way to always clean up stale entries thinger.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3913454154
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
Review Comment:
done
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3916938231
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
Review Comment:
> but we don't set a timer to send a refresh_token event until we get that response.
> not sure how to make it clearer, I agree it's subtle. I could use another atom as the default value for token_updated_at but it would never need to be referred to again in the code so that seems odd.
What if we don't get a successful response but a 500? We reset and fire another refresh later, that comes to the same handler compares last_used > token_update_at (=undefined) so it's false, we "let it expire". It expires, job crashes and restarts. When registering it sees the existing entry so it goes back to refresh and it goes into a loop?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3917061551
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,793 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now_secs()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ false ->
+ %% let it expire
+ ok
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, fin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, {unexpected_status_code, StatusCode}});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ couch_log:warning("NO MATCH", []),
Review Comment:
Debug line?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3825557588
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
Review Comment:
Should we bump use here since we're using it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3941754796
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,699 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ acquire_mref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ token_updated_at = NowSecs
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{acquire_mref = AcquireMRef} when AcquireMRef /= undefined ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
Review Comment:
It looks a bit better as the undefined comparison is gone however if in case of a 500 triggered refresh we'd still go into a loop and get stuck as last_used > token_updated_at will always stay false?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3605787671
##########
src/couch_replicator/src/couch_replicator_auth_ibm_iam.erl:
##########
@@ -0,0 +1,178 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_auth_ibm_iam).
+
+-behaviour(couch_replicator_auth).
+
+-export([
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+-export([token_acquirer/1]).
+
+-define(EARLY_REFRESH_MS, 5 * 60 * 1000).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include_lib("ibrowse/include/ibrowse.hrl").
+
+-record(state, {
+ acquirer,
+ apikey :: binary(),
+ refresh_at_ms,
+ token :: list()
+}).
+
+%% callbacks
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_apikey(HttpDb) of
+ ignore ->
+ ignore;
+ {ok, APIKey} ->
+ Acquirer = spawn_link(?MODULE, token_acquirer, [APIKey]),
Review Comment:
I had that earlier but it entailed duplication of the 'get token' ibrowse code (one synchronous, the other not). so I simplified to one version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#issuecomment-5027761271
I tried adding a supervisor-level API for replicator auth plugins https://github.com/apache/couchdb/pull/6070
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3606824856
##########
src/couch_replicator/src/couch_replicator_auth_ibm_iam.erl:
##########
@@ -0,0 +1,178 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_auth_ibm_iam).
+
+-behaviour(couch_replicator_auth).
+
+-export([
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+-export([token_acquirer/1]).
+
+-define(EARLY_REFRESH_MS, 5 * 60 * 1000).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include_lib("ibrowse/include/ibrowse.hrl").
+
+-record(state, {
+ acquirer,
+ apikey :: binary(),
+ refresh_at_ms,
+ token :: list()
+}).
+
+%% callbacks
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_apikey(HttpDb) of
+ ignore ->
+ ignore;
+ {ok, APIKey} ->
+ Acquirer = spawn_link(?MODULE, token_acquirer, [APIKey]),
Review Comment:
That's why I think we can just use a similar structure as the session plugin and always use the synchronous version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3917234239
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,793 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now_secs()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ false ->
+ %% let it expire
+ ok
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, fin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, {unexpected_status_code, StatusCode}});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ couch_log:warning("NO MATCH", []),
Review Comment:
oops, fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r4132723936
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,719 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ last_used,
+ last_updated,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ gen_server:cast(?MODULE, {update_last_used, APIKeyMAC}),
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ last_updated = NowSecs
Review Comment:
done.
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,719 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ last_used,
+ last_updated,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ gen_server:cast(?MODULE, {update_last_used, APIKeyMAC}),
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ NowSecs = now_secs(),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ last_used = NowSecs,
+ last_updated = NowSecs
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ ok;
+ true ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.last_used >= Entry#private_entry.last_updated of
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r4064663866
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,708 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{acquire_mref = AcquireMRef} when AcquireMRef /= undefined ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({used_since_acquisition, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.used_since_acquisition, true}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.used_since_acquisition of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ AcquireMRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{acquire_mref = AcquireMRef});
+ false ->
+ %% let it expire and free any waiters
+ reply_and_reset(Entry, {error, token_expiring})
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ demonitor(Entry#private_entry.acquire_mref, [flush]);
+ true ->
+ ok
+ end,
+ reply_all(Entry#private_entry.waiters, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(refresh_expire_check, #state{} = State) ->
+ NowSecs = now_secs(),
+ EarlyRefreshSecs = early_refresh_secs(),
+ Matches = ets:select(?PRIVATE, [
+ {
+ #private_entry{api_key_mac = '$1', expires_at_secs = '$2', _ = '_'},
+ [{'>=', '$2', NowSecs - EarlyRefreshSecs}],
Review Comment:
I wonder if this is right? I think we'd want to find all entries for which `Now >= ExpiresAt - EarlyRefreshSec` that is equivalent to `ExpiresAt =< Now + EarlyRefreshSec`
Then another thing to worry about is what happens if/when expires_at is not an integer but `undefined`
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,708 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
Review Comment:
I think this is still an issue. On first acquire failure the error sticks around and become persistent. A job may crash then restart, re-registers and will find an entry already and repeat the cycle.
Maybe the name is a bit mismatched. It seems like it should be something like "has anyone wanted a token since we handed one out already?" so if it's renamed as `wanted _since_token` it could match better with its use?
On register we started as `true` (someone wanted a token), acquire succeeds (clear, we satisfied whoever wanted a token) set to false, then a message like `{wanted_since_token, MAC}` from update_headers and we check if should refresh. So then only thing that sets it to `false` is a successful acquire
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3786984598
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
+
+get_token(APIKeyMAC) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} = Entry when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} = Entry ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
Review Comment:
hm, yes, perhaps. will add a clause in case (an http 204, say. IAM don't say they send it, but we should handle every possibility)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3797181695
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
Review Comment:
workers will update the auth gen_server at most one a minute if they are still using the token, I then decline to refresh the token if it hasn't been used since last refresh.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3825625593
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
Review Comment:
Will this always work? If either last_used or token_update_at might be undefined or a non-integer we might get unexpected states here. Monotonic times also always start with negative values. It should be fine to compare them as both of them came from the same monotonic clock and are initialized from it.
##########
src/docs/src/replication/replicator.rst:
##########
@@ -809,6 +809,28 @@ they are used. If they are not, then URL userinfo is checked. If credentials
are found there, then those credentials are used, otherwise basic auth header
is used.
+Using an IBM IAM apikey
+=======================
+
+If you've enabled the optional `couch_replicator_auth_ibm_auth`
Review Comment:
Extra `_auth` at the end
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
Review Comment:
Should we bump use here since we're using it the key?
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
Review Comment:
Minor nit: avoid overriding bult-ins. `now_sec()` is not terribly longer. And besides now traditionally returns a `{mega, sec, usec}` tuple so the code looks like we'd be handling that value.
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ false ->
+ %% let it expire
+ ok
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, fin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, {unexpected_status_code, StatusCode}});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, nofin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{
+ gun_body = [Data | Entry#private_entry.gun_body]
+ });
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_data, GunPid, GunStreamRef, fin, Data}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ResponseBody = lists:reverse([Data | Entry#private_entry.gun_body]),
+ case Entry#private_entry.gun_status_code of
+ 200 ->
+ case decode_iam_response(ResponseBody) of
+ {ok, Token, ExpiresInMs} ->
+ UUID = api_key_uuid(Token),
+ couch_log:notice("~p: refreshed api key ~s", [
+ ?MODULE, UUID
+ ]),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+
+ RefreshRef = erlang:send_after(
+ max(
+ ?MIN_REFRESH_MS,
+ ExpiresInMs - ?EARLY_REFRESH_MS - rand:uniform(?JITTER_MS)
+ ),
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ ),
+ ExpiresRef = erlang:send_after(
+ ExpiresInMs,
+ self(),
+ {expire_api_key_entry, Entry#private_entry.api_key_mac}
+ ),
+ true = ets:insert(?PUBLIC, #public_entry{
+ api_key_mac = Entry#private_entry.api_key_mac,
+ token = Token
+ }),
+ reply_and_reset(
+ Entry#private_entry{
+ api_key_uuid = UUID,
+ expires_ref = ExpiresRef,
+ refresh_ref = RefreshRef,
+ token_updated_at = now()
+ },
+ {ok, Token}
+ );
+ {error, Reason} ->
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, Reason
+ ]),
+ reply_and_reset(Entry, {error, Reason})
+ end;
+ StatusCode ->
+ ErrorMessage = extract_error_message(ResponseBody),
+ couch_log:notice("~p: failed to refresh api key ~s: ~p", [
+ ?MODULE, Entry#private_entry.api_key_uuid, ErrorMessage
+ ]),
+ case StatusCode of
+ 500 ->
+ erlang:send_after(
+ ?MIN_REFRESH_MS,
+ self(),
+ {refresh_token, Entry#private_entry.api_key_mac}
+ );
+ _ ->
+ ok
+ end,
+ reply_and_reset(Entry, {error, ErrorMessage})
+ end;
+ [] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_error, GunPid, GunStreamRef, Reason}, #state{gun_pid = GunPid} = State) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info({gun_error, GunPid, Reason}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun error ~p", [?MODULE, Reason]),
+ {noreply, State};
+handle_info(
+ {'DOWN', GunMRef, process, GunPid, Reason}, #state{gun_pid = GunPid, gun_mref = GunMRef} = State
+) ->
+ couch_log:warning("~p: gun process crashed for reason: ~p", [?MODULE, Reason]),
+ handle_info(restart_gun, State#state{gun_pid = undefined, gun_mref = undefined});
+handle_info({gun_up, GunPid, _Protocol}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, closed, []}, #state{gun_pid = GunPid} = State) ->
+ {noreply, State};
+handle_info({gun_down, GunPid, _Protocol, Reason, KilledStreams}, #state{gun_pid = GunPid} = State) ->
+ couch_log:warning("~p: gun connection down for reason: ~p", [?MODULE, Reason]),
+ lists:foreach(
+ fun(GunStreamRef) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, Reason});
+ [] ->
+ ok
+ end
+ end,
+ KilledStreams
+ ),
+ {noreply, State};
+handle_info(restart_gun, State) ->
+ case start_gun(State) of
+ {ok, NewState} ->
+ {noreply, NewState};
+ {error, Reason} ->
+ couch_log:warning("~p: gun restart failed for reason: ~p", [?MODULE, Reason]),
+ erlang:send_after(5000, self(), restart_gun),
+ {noreply, State}
+ end;
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
+handle_info(token_url_change, State0) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ stop_gun(State0),
Review Comment:
Should we also demonitor and flush here? Stop will call gun:flush() but our monitor is separate from that machinery.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3936486413
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
Review Comment:
Wonder if something like this would make sense instead of the comparison
```erlang
% refresh if never got a token (updated_at=undefined), or any callers are waiting, or was used since last refresh
should_refresh(#private_entry{token_updated_at = undefined}) ->
true;
should_refresh(#private_entry{waiters = [_ | _]}) ->
true;
should_refresh(#private_entry{last_used = LastUsed, token_updated_at = TokenUpdatedAt}) ->
LastUsed > TokenUpdatedAt
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson merged PR #6069:
URL: https://github.com/apache/couchdb/pull/6069
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3913653753
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,750 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-compile({no_auto_import, [now/0]}).
+
+-define(EARLY_REFRESH_MS, 300_000).
+-define(JITTER_MS, 60_000).
+-define(MIN_REFRESH_MS, 10_000).
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
Review Comment:
last_used is initialized to current time on first entry but token_updated_at is undefined until we get the IAM response, but we don't set a timer to send a refresh_token event until we get that response. if that response is successful we set token_updated_at to the current time. if not, we don't set it, and so last_used > undefined is false for any value of last_used so we do nothing here, which is what we'd want.
not sure how to make it clearer, I agree it's subtle. I could use another atom as the default value for token_updated_at but it would never need to be referred to again in the code so that seems odd.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3917061551
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,793 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ gun_mref,
+ gun_pid,
+ mac_key,
+ token_uri_map
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ expires_ref,
+ gun_body = [],
+ gun_status_code,
+ gun_stream_ref,
+ last_used,
+ refresh_ref,
+ token_updated_at,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_last_used(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, token_timeout());
+ [] ->
+ {error, no_token}
+ end.
+
+update_last_used(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {update_last_used, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_uri_map() of
+ {ok, TokenURIMap} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ start_gun(#state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_uri_map = TokenURIMap
+ });
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ self() ! {update_last_used, Entry#private_entry.api_key_mac},
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ GunStreamRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ gun_stream_ref = GunStreamRef,
+ last_used = now_secs()
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{gun_stream_ref = GunStreamRef} when
+ GunStreamRef /= undefined
+ ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({update_last_used, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.last_used, now_secs()}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{gun_stream_ref = undefined} = Entry] ->
+ case Entry#private_entry.last_used > Entry#private_entry.token_updated_at of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ GunStreamRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{gun_stream_ref = GunStreamRef});
+ false ->
+ %% let it expire
+ ok
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ cancel_timer(Entry#private_entry.refresh_ref),
+ cancel_timer(Entry#private_entry.expires_ref),
+ reply_all(Entry, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, fin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ reply_and_reset(Entry, {error, {unexpected_status_code, StatusCode}});
+ _ ->
+ ok
+ end,
+ {noreply, State};
+handle_info(
+ {gun_response, GunPid, GunStreamRef, nofin, StatusCode, _Headers},
+ #state{gun_pid = GunPid} = State
+) ->
+ case match_on_gun_stream_ref(GunStreamRef) of
+ [#private_entry{} = Entry] ->
+ ets:insert(?PRIVATE, Entry#private_entry{gun_status_code = StatusCode});
+ _ ->
+ couch_log:warning("NO MATCH", []),
Review Comment:
Debug line? If not add a module name and some context
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "rnewson (via GitHub)" <gi...@apache.org>.
rnewson commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3786995418
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,673 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(EARLY_REFRESH_MS, 300000).
+-define(MIN_REFRESH_MS, 10000).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(state, {
+ token_uri_map,
+ mac_key,
+ gun_pid,
+ gun_mref
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key,
+ api_key_uuid,
+ api_key_mac,
+ gun_stream_ref,
+ gun_status_code,
+ gun_body = [],
+ refresh_ref,
+ expires_ref,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ application:ensure_all_started(gun),
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ {ok, HttpDb, APIKeyMAC};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(APIKeyMAC, Headers) when is_list(Headers) ->
+ case get_token(APIKeyMAC) of
+ {ok, Token} ->
+ {[{~"Authorization", <<"Bearer ", Token/binary>>} | Headers], APIKeyMAC};
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, APIKeyMAC}
+ end.
+
+handle_response(APIKeyMAC, _StatusCode, _Headers) ->
+ {continue, APIKeyMAC}.
+
+cleanup(_APIKeyMAC) ->
+ ok.
Review Comment:
... yes. the problem is I've no way to know when no replication job refers to a particular apikey. 'register' gets called multiple times per replication, which is why I ripped out my refcounting (which did ref-decrementing in cleanup and deleted entries when that hit 0).
perhaps I can add a 'last used' timestamp in the private table and just not renew an entry that hasn't been used for a while
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r3980423678
##########
rel/overlay/etc/default.ini:
##########
@@ -815,8 +815,9 @@ partitioned||* = true
; particular endpoint (source or target). Normally couch_replicator_auth_noop
; would be used at the end of the list as a "catch-all". It doesn't do anything
; and effectively implements the previous behavior of using basic auth.
-; There are currently two plugins available:
+; There are currently three plugins available:
; couch_replicator_auth_session - use _session cookie authentication
+; couch_replicator_auth_ibm - use IBM's IAM service for authentication
Review Comment:
Add the config options for it
##########
src/docs/src/replication/replicator.rst:
##########
@@ -809,6 +809,28 @@ they are used. If they are not, then URL userinfo is checked. If credentials
are found there, then those credentials are used, otherwise basic auth header
is used.
+Using an IBM IAM apikey
+=======================
+
+If you've enabled the optional `couch_replicator_auth_ibm`
Review Comment:
Maybe add a few config options in config docs section
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] couch replication auth plugin for IBM IAM with refresh [couchdb]
Posted by "nickva (via GitHub)" <gi...@apache.org>.
nickva commented on code in PR #6069:
URL: https://github.com/apache/couchdb/pull/6069#discussion_r4064148268
##########
src/couch_replicator/src/couch_replicator_auth_ibm.erl:
##########
@@ -0,0 +1,708 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module allows a replication source or target to use an IAM api key for authentication.
+%
+% Features;
+%
+% Automatic refreshing of time-limited token before expiration
+% Deduplication - only one token will be acquired for each distinct IAM api key
+%
+% Implementation details
+%
+% As api keys are sensitive, the only copy of api keys is held in a private ETS table
+% owned by this module's gen_server. An opaque reference is returned to clients (this is
+% a message authentication code where the key is a non-persisted value generated by the
+% gen_server)
+
+-module(couch_replicator_auth_ibm).
+
+-behaviour(couch_replicator_auth).
+-behaviour(gen_server).
+-behaviour(config_listener).
+
+-export([
+ sup_initialize/0,
+ sup_cleanup/1,
+ initialize/1,
+ update_headers/2,
+ handle_response/3,
+ cleanup/1
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ terminate/2
+]).
+
+% config_listener callbacks
+-export([
+ handle_config_change/5,
+ handle_config_terminate/3
+]).
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+-define(MINUTE, 60).
+-define(PUBLIC, couch_replicator_auth_ibm_public).
+-define(PRIVATE, couch_replicator_auth_ibm_private).
+
+-record(worker_state, {
+ api_key_mac,
+ last_used
+}).
+
+-record(state, {
+ mac_key,
+ token_url
+}).
+
+-record(public_entry, {
+ api_key_mac,
+ token
+}).
+
+-record(private_entry, {
+ api_key_mac,
+ api_key_uuid,
+ api_key,
+ % Number of seconds (counted since January 1st, 1970) until the IAM access token will expire.
+ expires_at_secs,
+ acquire_mref,
+ used_since_acquisition,
+ waiters = []
+}).
+
+%% callbacks
+
+sup_initialize() ->
+ {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+sup_cleanup(_) ->
+ ok = gen_server:stop(?MODULE).
+
+initialize(#httpdb{} = HttpDb) ->
+ case extract_api_key(HttpDb) of
+ {ok, APIKey} ->
+ case gen_server:call(?MODULE, {register, APIKey}) of
+ {ok, APIKeyMAC} ->
+ WorkerState = #worker_state{api_key_mac = APIKeyMAC, last_used = now_secs()},
+ {ok, HttpDb, WorkerState};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, _} ->
+ ignore
+ end.
+
+update_headers(WorkerState, Headers) when is_list(Headers) ->
+ case get_token(WorkerState) of
+ {ok, Token} ->
+ {
+ [{~"Authorization", <<"Bearer ", Token/binary>>} | Headers],
+ update_used_since_acquisition(WorkerState)
+ };
+ {error, Reason} ->
+ couch_log:warning("Error when retrieving token: ~p", [Reason]),
+ {Headers, WorkerState}
+ end.
+
+handle_response(WorkerState, _StatusCode, _Headers) ->
+ {continue, WorkerState}.
+
+cleanup(_WorkerState) ->
+ ok.
+
+get_token(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC} = WorkerState,
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [#public_entry{token = Token}] when is_binary(Token) ->
+ {ok, Token};
+ [#public_entry{}] ->
+ gen_server:call(?MODULE, {get_token, APIKeyMAC}, request_timeout_ms());
+ [] ->
+ {error, no_token}
+ end.
+
+update_used_since_acquisition(#worker_state{} = WorkerState) ->
+ #worker_state{api_key_mac = APIKeyMAC, last_used = LastUsed} = WorkerState,
+ Now = now_secs(),
+ %% only send a message once per minute at most.
+ case Now - LastUsed > ?MINUTE of
+ true ->
+ ?MODULE ! {used_since_acquisition, APIKeyMAC},
+ WorkerState#worker_state{last_used = Now};
+ false ->
+ WorkerState
+ end.
+
+%% gen_server callbacks.
+
+init(_) ->
+ case token_url() of
+ {ok, TokenUrl} ->
+ ?PUBLIC = ets:new(?PUBLIC, [protected, {keypos, #public_entry.api_key_mac}, named_table]),
+ ?PRIVATE = ets:new(?PRIVATE, [
+ private, {keypos, #private_entry.api_key_mac}, named_table
+ ]),
+ ok = config:listen_for_changes(?MODULE, nil),
+ erlang:send_after(check_interval_ms(), self(), refresh_expire_check),
+ {ok, #state{
+ mac_key = crypto:strong_rand_bytes(32),
+ token_url = TokenUrl
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+handle_call({register, APIKey}, _From, State) ->
+ case ets:match_object(?PRIVATE, #private_entry{api_key = APIKey, _ = '_'}) of
+ [#private_entry{} = Entry] ->
+ {reply, {ok, Entry#private_entry.api_key_mac}, State};
+ [] ->
+ AcquireMRef = acquire_token(APIKey, State),
+ APIKeyMAC = mac(State#state.mac_key, APIKey),
+ true = ets:insert_new(?PUBLIC, #public_entry{
+ api_key_mac = APIKeyMAC
+ }),
+ true = ets:insert_new(?PRIVATE, #private_entry{
+ api_key = APIKey,
+ api_key_mac = APIKeyMAC,
+ acquire_mref = AcquireMRef,
+ used_since_acquisition = false
+ }),
+ {reply, {ok, APIKeyMAC}, State}
+ end;
+handle_call({get_token, APIKeyMAC}, From, State) ->
+ case ets:lookup(?PUBLIC, APIKeyMAC) of
+ [] ->
+ {reply, {error, no_such_api_key}, State};
+ [#public_entry{token = Token}] when Token /= undefined ->
+ {reply, {ok, Token}, State};
+ [#public_entry{}] ->
+ [#private_entry{} = Entry] = ets:lookup(?PRIVATE, APIKeyMAC),
+ ets:insert(?PRIVATE, Entry#private_entry{waiters = [From | Entry#private_entry.waiters]}),
+ case Entry of
+ #private_entry{acquire_mref = AcquireMRef} when AcquireMRef /= undefined ->
+ ok;
+ #private_entry{} ->
+ self() ! {refresh_token, APIKeyMAC}
+ end,
+ {noreply, State}
+ end;
+handle_call(_Msg, _From, State) ->
+ {reply, {error, unexpected_msg}, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({used_since_acquisition, APIKeyMAC}, State) ->
+ ets:update_element(?PRIVATE, APIKeyMAC, {#private_entry.used_since_acquisition, true}),
+ {noreply, State};
+handle_info({refresh_token, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{acquire_mref = undefined} = Entry] ->
+ case Entry#private_entry.used_since_acquisition of
+ true ->
+ couch_log:notice("~p: refreshing api key ~s", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ AcquireMRef = acquire_token(Entry#private_entry.api_key, State),
+ ets:insert(?PRIVATE, Entry#private_entry{acquire_mref = AcquireMRef});
+ false ->
+ %% let it expire and free any waiters
+ reply_and_reset(Entry, {error, token_expiring})
+ end;
+ [#private_entry{}] ->
+ ok
+ end,
+ {noreply, State};
+handle_info({expire_api_key_entry, APIKeyMAC}, State) ->
+ case ets:lookup(?PRIVATE, APIKeyMAC) of
+ [] ->
+ ok;
+ [#private_entry{} = Entry] ->
+ couch_log:warning("~p: api key entry ~s passed expiration time", [
+ ?MODULE, Entry#private_entry.api_key_uuid
+ ]),
+ ets:delete(?PUBLIC, APIKeyMAC),
+ ets:delete(?PRIVATE, APIKeyMAC),
+ if
+ Entry#private_entry.acquire_mref /= undefined ->
+ demonitor(Entry#private_entry.acquire_mref, [flush]);
+ true ->
+ ok
+ end,
+ reply_all(Entry#private_entry.waiters, {error, expired_api_key_entry})
+ end,
+ {noreply, State};
+handle_info(refresh_expire_check, #state{} = State) ->
+ NowSecs = now_secs(),
+ EarlyRefreshSecs = early_refresh_secs(),
+ Matches = ets:select(?PRIVATE, [
+ {
+ #private_entry{api_key_mac = '$1', expires_at_secs = '$2', _ = '_'},
+ [{'>=', '$2', NowSecs - EarlyRefreshSecs}],
+ [['$1', '$2']]
+ }
+ ]),
+ lists:foreach(
+ fun([APIKeyMAC, ExpiresAtSecs]) ->
+ if
+ NowSecs >= ExpiresAtSecs ->
+ self() ! {expire_api_key_entry, APIKeyMAC};
+ true ->
+ erlang:send_after(jitter_ms(), self(), {refresh_token, APIKeyMAC})
Review Comment:
for it to be jitter we'd probably want some randomness added
maybe something like
```erlang
case config:get_integer(...) of
0 -> 0;
J -> rand:uniform(J)
end
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org