You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2020/04/27 13:15:32 UTC
[couchdb] 01/01: Add native encryption support
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch aegis
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 67038b9573c80d8262e7e58877b8faabb24fab29
Author: Robert Newson <rn...@apache.org>
AuthorDate: Wed Apr 8 16:40:26 2020 +0100
Add native encryption support
A new application, aegis, is introduced to provide strong at-rest
protection of CouchDB data (where possible).
Currently we encrypt the following values (if enabled):
1. Document content
2. Attachment content
3. Index values
Things not encrypted:
1. _all_docs
2. _changes
3. doc id
4. doc rev
5. Index keys
6. All other metadata
Co-Authored-By: eiri@apache.org
Co-Authored-By: rnewson@apache.org
---
configure | 19 ++
rebar.config.script | 1 +
rel/reltool.config | 2 +
src/aegis/rebar.config.script | 35 +++
.../src/fabric.app.src => aegis/src/aegis.app.src} | 43 ++--
src/aegis/src/aegis.erl | 72 ++++++
src/aegis/src/aegis.hrl | 57 +++++
.../src/fabric.app.src => aegis/src/aegis_app.erl} | 35 ++-
.../src/aegis_key_manager.erl} | 31 +--
.../src/aegis_keywrap.erl} | 34 ++-
.../src/aegis_noop_key_manager.erl} | 40 ++-
src/aegis/src/aegis_server.erl | 275 +++++++++++++++++++++
.../src/fabric.app.src => aegis/src/aegis_sup.erl} | 55 +++--
.../test/aegis_basic_test.erl} | 26 +-
src/aegis/test/aegis_server_test.erl | 165 +++++++++++++
src/chttpd/src/chttpd.erl | 8 +
src/couch_views/src/couch_views_fdb.erl | 8 +-
src/fabric/include/fabric2.hrl | 1 +
src/fabric/src/fabric.app.src | 3 +-
src/fabric/src/fabric2_fdb.erl | 31 ++-
20 files changed, 777 insertions(+), 164 deletions(-)
diff --git a/configure b/configure
index 854366c..b91b18d 100755
--- a/configure
+++ b/configure
@@ -96,6 +96,24 @@ parse_opts() {
continue
;;
+ --key-manager)
+ if [ -n "$2" ]; then
+ eval AEGIS_KEY_MANAGER=$2
+ shift 2
+ continue
+ else
+ printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2
+ exit 1
+ fi
+ ;;
+ --key-manager=?*)
+ eval AEGIS_KEY_MANAGER=${1#*=}
+ ;;
+ --key-manager=)
+ printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2
+ exit 1
+ ;;
+
--dev)
WITH_DOCS=0
WITH_FAUXTON=0
@@ -241,6 +259,7 @@ cat > $rootdir/config.erl << EOF
{with_curl, $WITH_CURL}.
{with_proper, $WITH_PROPER}.
{erlang_md5, $ERLANG_MD5}.
+{aegis_key_manager, "$AEGIS_KEY_MANAGER"}.
{spidermonkey_version, "$SM_VSN"}.
EOF
diff --git a/rebar.config.script b/rebar.config.script
index b3ea2c9..2badaba 100644
--- a/rebar.config.script
+++ b/rebar.config.script
@@ -135,6 +135,7 @@ SubDirs = [
"src/ddoc_cache",
"src/dreyfus",
"src/fabric",
+ "src/aegis",
"src/couch_jobs",
"src/couch_expiring_cache",
"src/global_changes",
diff --git a/rel/reltool.config b/rel/reltool.config
index 9fbf285..b59c95f 100644
--- a/rel/reltool.config
+++ b/rel/reltool.config
@@ -27,6 +27,7 @@
syntax_tools,
xmerl,
%% couchdb
+ aegis,
b64url,
bear,
chttpd,
@@ -90,6 +91,7 @@
{app, xmerl, [{incl_cond, include}]},
%% couchdb
+ {app, aegis, [{incl_cond, include}]},
{app, b64url, [{incl_cond, include}]},
{app, bear, [{incl_cond, include}]},
{app, chttpd, [{incl_cond, include}]},
diff --git a/src/aegis/rebar.config.script b/src/aegis/rebar.config.script
new file mode 100644
index 0000000..ef148bf
--- /dev/null
+++ b/src/aegis/rebar.config.script
@@ -0,0 +1,35 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+CouchConfig = case filelib:is_file(os:getenv("COUCHDB_CONFIG")) of
+ true ->
+ {ok, Result} = file:consult(os:getenv("COUCHDB_CONFIG")),
+ Result;
+ false ->
+ []
+end.
+
+AegisKeyManager = case lists:keyfind(aegis_key_manager, 1, CouchConfig) of
+ {aegis_key_manager, Module} when Module /= "" ->
+ list_to_atom(Module);
+ _ ->
+ aegis_noop_key_manager
+end,
+
+CurrentOpts = case lists:keyfind(erl_opts, 1, CONFIG) of
+ {erl_opts, Opts} -> Opts;
+ false -> []
+end,
+
+AegisOpts = {d, 'AEGIS_KEY_MANAGER', AegisKeyManager},
+lists:keystore(erl_opts, 1, CONFIG, {erl_opts, [AegisOpts | CurrentOpts]}).
diff --git a/src/fabric/src/fabric.app.src b/src/aegis/src/aegis.app.src
similarity index 57%
copy from src/fabric/src/fabric.app.src
copy to src/aegis/src/aegis.app.src
index 0538b19..deb1526 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/aegis/src/aegis.app.src
@@ -10,24 +10,25 @@
% License for the specific language governing permissions and limitations under
% the License.
-{application, fabric, [
- {description, "Routing and proxying layer for CouchDB cluster"},
- {vsn, git},
- {mod, {fabric2_app, []}},
- {registered, [
- fabric_server
- ]},
- {applications, [
- kernel,
- stdlib,
- config,
- couch_epi,
- couch,
- ctrace,
- rexi,
- mem3,
- couch_log,
- couch_stats,
- erlfdb
- ]}
-]}.
+{application, aegis,
+ [
+ {description, "If it's good enough for Zeus, it's good enough for CouchDB"},
+ {vsn, git},
+ {mod, {aegis_app, []}},
+ {registered, [
+ aegis_server
+ ]},
+ {applications,
+ [kernel,
+ stdlib,
+ crypto,
+ couch_log,
+ erlfdb
+ ]},
+ {env,[]},
+ {modules, []},
+ {maintainers, []},
+ {licenses, []},
+ {links, []}
+ ]
+}.
diff --git a/src/aegis/src/aegis.erl b/src/aegis/src/aegis.erl
new file mode 100644
index 0000000..e8a0b4b
--- /dev/null
+++ b/src/aegis/src/aegis.erl
@@ -0,0 +1,72 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(aegis).
+-include_lib("fabric/include/fabric2.hrl").
+
+
+-define(WRAPPED_KEY, {?DB_AEGIS, 1}).
+
+
+-export([
+ init_db/2,
+ open_db/1,
+
+ decrypt/2,
+ decrypt/3,
+ encrypt/3,
+ wrap_fold_fun/2
+]).
+
+init_db(#{} = Db, Options) ->
+ Db#{
+ is_encrypted => aegis_server:init_db(Db, Options)
+ }.
+
+
+open_db(#{} = Db) ->
+ Db#{
+ is_encrypted => aegis_server:open_db(Db)
+ }.
+
+
+encrypt(#{} = _Db, _Key, <<>>) ->
+ <<>>;
+
+encrypt(#{is_encrypted := false}, _Key, Value) when is_binary(Value) ->
+ Value;
+
+encrypt(#{is_encrypted := true} = Db, Key, Value)
+ when is_binary(Key), is_binary(Value) ->
+ aegis_server:encrypt(Db, Key, Value).
+
+
+decrypt(#{} = Db, Rows) when is_list(Rows) ->
+ lists:map(fun({Key, Value}) ->
+ {Key, decrypt(Db, Key, Value)}
+ end, Rows).
+
+decrypt(#{} = _Db, _Key, <<>>) ->
+ <<>>;
+
+decrypt(#{is_encrypted := false}, _Key, Value) when is_binary(Value) ->
+ Value;
+
+decrypt(#{is_encrypted := true} = Db, Key, Value)
+ when is_binary(Key), is_binary(Value) ->
+ aegis_server:decrypt(Db, Key, Value).
+
+
+wrap_fold_fun(Db, Fun) when is_function(Fun, 2) ->
+ fun({Key, Value}, Acc) ->
+ Fun({Key, decrypt(Db, Key, Value)}, Acc)
+ end.
diff --git a/src/aegis/src/aegis.hrl b/src/aegis/src/aegis.hrl
new file mode 100644
index 0000000..2a2a2dc
--- /dev/null
+++ b/src/aegis/src/aegis.hrl
@@ -0,0 +1,57 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+%% Assume old crypto api
+
+-define(sha256_hmac(Key, PlainText), crypto:hmac(sha256, Key, PlainText)).
+
+-define(aes_gcm_encrypt(Key, IV, AAD, Data),
+ crypto:block_encrypt(aes_gcm, Key, IV, {AAD, Data, 16})).
+
+-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag),
+ crypto:block_decrypt(aes_gcm, Key, IV, {AAD, CipherText, CipherTag})).
+
+-define(aes_ecb_encrypt(Key, Data),
+ crypto:block_encrypt(aes_ecb, Key, Data)).
+
+-define(aes_ecb_decrypt(Key, Data),
+ crypto:block_decrypt(aes_ecb, Key, Data)).
+
+%% Replace macros if new crypto api is available
+-ifdef(OTP_RELEASE).
+-if(?OTP_RELEASE >= 22).
+
+-undef(sha256_hmac).
+-define(sha256_hmac(Key, PlainText), crypto:mac(hmac, sha256, Key, PlainText)).
+
+-undef(aes_gcm_encrypt).
+-define(aes_gcm_encrypt(Key, IV, AAD, Data),
+ crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, Data, AAD, 16, true)).
+
+-undef(aes_gcm_decrypt).
+-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag),
+ crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, CipherText,
+ AAD, CipherTag, false)).
+
+-define(key_alg(Key), case bit_size(Key) of
+ 128 -> aes_128_ecb; 192 -> aes_192_ecb; 256 -> aes_256_ecb end).
+
+-undef(aes_ecb_encrypt).
+-define(aes_ecb_encrypt(Key, Data),
+ crypto:crypto_one_time(?key_alg(Key), Key, Data, true)).
+
+-undef(aes_ecb_decrypt).
+-define(aes_ecb_decrypt(Key, Data),
+ crypto:crypto_one_time(?key_alg(Key), Key, Data, false)).
+
+-endif.
+-endif.
\ No newline at end of file
diff --git a/src/fabric/src/fabric.app.src b/src/aegis/src/aegis_app.erl
similarity index 57%
copy from src/fabric/src/fabric.app.src
copy to src/aegis/src/aegis_app.erl
index 0538b19..4a5a11f 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/aegis/src/aegis_app.erl
@@ -10,24 +10,17 @@
% License for the specific language governing permissions and limitations under
% the License.
-{application, fabric, [
- {description, "Routing and proxying layer for CouchDB cluster"},
- {vsn, git},
- {mod, {fabric2_app, []}},
- {registered, [
- fabric_server
- ]},
- {applications, [
- kernel,
- stdlib,
- config,
- couch_epi,
- couch,
- ctrace,
- rexi,
- mem3,
- couch_log,
- couch_stats,
- erlfdb
- ]}
-]}.
+-module(aegis_app).
+
+-behaviour(application).
+
+
+-export([start/2, stop/1]).
+
+
+start(_StartType, _StartArgs) ->
+ aegis_sup:start_link().
+
+
+stop(_State) ->
+ ok.
diff --git a/src/fabric/src/fabric.app.src b/src/aegis/src/aegis_key_manager.erl
similarity index 57%
copy from src/fabric/src/fabric.app.src
copy to src/aegis/src/aegis_key_manager.erl
index 0538b19..aa9e342 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/aegis/src/aegis_key_manager.erl
@@ -10,24 +10,13 @@
% License for the specific language governing permissions and limitations under
% the License.
-{application, fabric, [
- {description, "Routing and proxying layer for CouchDB cluster"},
- {vsn, git},
- {mod, {fabric2_app, []}},
- {registered, [
- fabric_server
- ]},
- {applications, [
- kernel,
- stdlib,
- config,
- couch_epi,
- couch,
- ctrace,
- rexi,
- mem3,
- couch_log,
- couch_stats,
- erlfdb
- ]}
-]}.
+-module(aegis_key_manager).
+
+
+
+-callback init_db(
+ Db :: #{},
+ DbOptions :: list()) -> {ok, binary()} | false.
+
+
+-callback open_db(Db :: #{}) -> {ok, binary()} | false.
diff --git a/src/couch/src/couch_keywrap.erl b/src/aegis/src/aegis_keywrap.erl
similarity index 84%
rename from src/couch/src/couch_keywrap.erl
rename to src/aegis/src/aegis_keywrap.erl
index 0d1e3f5..58c7668 100644
--- a/src/couch/src/couch_keywrap.erl
+++ b/src/aegis/src/aegis_keywrap.erl
@@ -1,4 +1,17 @@
--module(couch_keywrap).
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(aegis_keywrap).
+-include("aegis.hrl").
%% Implementation of NIST Special Publication 800-38F
%% For wrapping and unwrapping keys with AES.
@@ -7,25 +20,6 @@
-define(ICV1, 16#A6A6A6A6A6A6A6A6).
-%% Assume old crypto api
--define(aes_ecb_encrypt(Key, Data),
- crypto:block_encrypt(aes_ecb, Key, Data)).
--define(aes_ecb_decrypt(Key, Data),
- crypto:block_decrypt(aes_ecb, Key, Data)).
-
-%% Replace macros if new crypto api is available
--ifdef(OTP_RELEASE).
--if(?OTP_RELEASE >= 22).
--define(key_alg(Key), case bit_size(Key) of 128 -> aes_128_ecb; 192 -> aes_192_ecb; 256 -> aes_256_ecb end).
--undef(aes_ecb_encrypt).
--define(aes_ecb_encrypt(Key, Data),
- crypto:crypto_one_time(?key_alg(Key), Key, Data, true)).
--undef(aes_ecb_decrypt).
--define(aes_ecb_decrypt(Key, Data),
- crypto:crypto_one_time(?key_alg(Key), Key, Data, false)).
--endif.
--endif.
-
-spec key_wrap(WrappingKey :: binary(), KeyToWrap :: binary()) -> binary().
key_wrap(WrappingKey, KeyToWrap)
when is_binary(WrappingKey), bit_size(KeyToWrap) rem 64 == 0 ->
diff --git a/src/fabric/src/fabric.app.src b/src/aegis/src/aegis_noop_key_manager.erl
similarity index 57%
copy from src/fabric/src/fabric.app.src
copy to src/aegis/src/aegis_noop_key_manager.erl
index 0538b19..2b61f1d 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/aegis/src/aegis_noop_key_manager.erl
@@ -10,24 +10,22 @@
% License for the specific language governing permissions and limitations under
% the License.
-{application, fabric, [
- {description, "Routing and proxying layer for CouchDB cluster"},
- {vsn, git},
- {mod, {fabric2_app, []}},
- {registered, [
- fabric_server
- ]},
- {applications, [
- kernel,
- stdlib,
- config,
- couch_epi,
- couch,
- ctrace,
- rexi,
- mem3,
- couch_log,
- couch_stats,
- erlfdb
- ]}
-]}.
+-module(aegis_noop_key_manager).
+
+
+-behaviour(aegis_key_manager).
+
+
+-export([
+ init_db/2,
+ open_db/1
+]).
+
+
+
+init_db(#{} = _Db, _Options) ->
+ false.
+
+
+open_db(#{} = _Db) ->
+ false.
diff --git a/src/aegis/src/aegis_server.erl b/src/aegis/src/aegis_server.erl
new file mode 100644
index 0000000..be8202c
--- /dev/null
+++ b/src/aegis/src/aegis_server.erl
@@ -0,0 +1,275 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(aegis_server).
+
+-behaviour(gen_server).
+
+-vsn(1).
+
+
+-include("aegis.hrl").
+
+
+%% aegis_server API
+-export([
+ start_link/0,
+ init_db/2,
+ open_db/1,
+ encrypt/3,
+ decrypt/3
+]).
+
+%% gen_server callbacks
+-export([
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
+]).
+
+
+
+-define(KEY_CHECK, aegis_key_check).
+-define(INIT_TIMEOUT, 60000).
+-define(TIMEOUT, 10000).
+
+
+-record(entry, {uuid, encryption_key}).
+
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+-spec init_db(Db :: #{}, Options :: list()) -> boolean().
+init_db(#{uuid := UUID} = Db, Options) ->
+ process_flag(sensitive, true),
+
+ case ?AEGIS_KEY_MANAGER:init_db(Db, Options) of
+ {ok, DbKey} ->
+ gen_server:call(?MODULE, {insert_key, UUID, DbKey}),
+ true;
+ false ->
+ false
+ end.
+
+
+-spec open_db(Db :: #{}) -> boolean().
+open_db(#{} = Db) ->
+ process_flag(sensitive, true),
+
+ case do_open_db(Db) of
+ {ok, _DbKey} ->
+ true;
+ false ->
+ false
+ end.
+
+
+-spec encrypt(Db :: #{}, Key :: binary(), Value :: binary()) -> binary().
+encrypt(#{} = Db, Key, Value) when is_binary(Key), is_binary(Value) ->
+ #{
+ uuid := UUID
+ } = Db,
+
+ case ets:member(?KEY_CHECK, UUID) of
+ true ->
+ case gen_server:call(?MODULE, {encrypt, Db, Key, Value}) of
+ CipherText when is_binary(CipherText) ->
+ CipherText;
+ {error, {_Tag, {_C_FileName,_LineNumber}, _Desc} = Reason} ->
+ couch_log:error("aegis encryption failure: ~p ", [Reason]),
+ erlang:error(decryption_failed);
+ {error, Reason} ->
+ erlang:error(Reason)
+ end;
+ false ->
+ process_flag(sensitive, true),
+
+ {ok, DbKey} = do_open_db(Db),
+ do_encrypt(DbKey, Db, Key, Value)
+ end.
+
+
+-spec decrypt(Db :: #{}, Key :: binary(), Value :: binary()) -> binary().
+decrypt(#{} = Db, Key, Value) when is_binary(Key), is_binary(Value) ->
+ #{
+ uuid := UUID
+ } = Db,
+
+ case ets:member(?KEY_CHECK, UUID) of
+ true ->
+ case gen_server:call(?MODULE, {decrypt, Db, Key, Value}) of
+ PlainText when is_binary(PlainText) ->
+ PlainText;
+ {error, {_Tag, {_C_FileName,_LineNumber}, _Desc} = Reason} ->
+ couch_log:error("aegis decryption failure: ~p ", [Reason]),
+ erlang:error(decryption_failed);
+ {error, Reason} ->
+ erlang:error(Reason)
+ end;
+ false ->
+ process_flag(sensitive, true),
+
+ {ok, DbKey} = do_open_db(Db),
+ do_decrypt(DbKey, Db, Key, Value)
+ end.
+
+
+%% gen_server functions
+
+init([]) ->
+ process_flag(sensitive, true),
+ Cache = ets:new(?MODULE, [set, private, {keypos, #entry.uuid}]),
+ ets:new(?KEY_CHECK, [named_table, protected, {read_concurrency, true}]),
+
+ St = #{
+ cache => Cache
+ },
+ {ok, St, ?INIT_TIMEOUT}.
+
+
+terminate(_Reason, _St) ->
+ ok.
+
+
+handle_call({insert_key, UUID, DbKey}, _From, #{cache := Cache} = St) ->
+ ok = insert(Cache, UUID, DbKey),
+ {reply, ok, St, ?TIMEOUT};
+
+handle_call({encrypt, #{uuid := UUID} = Db, Key, Value}, From, St) ->
+ #{
+ cache := Cache
+ } = St,
+
+ {ok, DbKey} = lookup(Cache, UUID),
+
+ erlang:spawn(fun() ->
+ process_flag(sensitive, true),
+ try
+ do_encrypt(DbKey, Db, Key, Value)
+ of
+ Resp ->
+ gen_server:reply(From, Resp)
+ catch
+ _:Error ->
+ gen_server:reply(From, {error, Error})
+ end
+ end),
+
+ {noreply, St, ?TIMEOUT};
+
+handle_call({decrypt, #{uuid := UUID} = Db, Key, Value}, From, St) ->
+ #{
+ cache := Cache
+ } = St,
+
+ {ok, DbKey} = lookup(Cache, UUID),
+
+ erlang:spawn(fun() ->
+ process_flag(sensitive, true),
+ try
+ do_decrypt(DbKey, Db, Key, Value)
+ of
+ Resp ->
+ gen_server:reply(From, Resp)
+ catch
+ _:Error ->
+ gen_server:reply(From, {error, Error})
+ end
+ end),
+
+ {noreply, St, ?TIMEOUT};
+
+handle_call(_Msg, _From, St) ->
+ {noreply, St}.
+
+
+handle_cast(_Msg, St) ->
+ {noreply, St}.
+
+
+handle_info(_Msg, St) ->
+ {noreply, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+ {ok, St}.
+
+
+%% private functions
+
+do_open_db(#{uuid := UUID} = Db) ->
+ case ?AEGIS_KEY_MANAGER:open_db(Db) of
+ {ok, DbKey} ->
+ gen_server:call(?MODULE, {insert_key, UUID, DbKey}),
+ {ok, DbKey};
+ false ->
+ false
+ end.
+
+
+do_encrypt(DbKey, #{uuid := UUID}, Key, Value) ->
+ EncryptionKey = crypto:strong_rand_bytes(32),
+ <<WrappedKey:320>> = aegis_keywrap:key_wrap(DbKey, EncryptionKey),
+
+ {CipherText, <<CipherTag:128>>} =
+ ?aes_gcm_encrypt(
+ EncryptionKey,
+ <<0:96>>,
+ <<UUID/binary, 0:8, Key/binary>>,
+ Value),
+ <<1:8, WrappedKey:320, CipherTag:128, CipherText/binary>>.
+
+
+do_decrypt(DbKey, #{uuid := UUID}, Key, Value) ->
+ case Value of
+ <<1:8, WrappedKey:320, CipherTag:128, CipherText/binary>> ->
+ case aegis_keywrap:key_unwrap(DbKey, <<WrappedKey:320>>) of
+ fail ->
+ erlang:error(decryption_failed);
+ DecryptionKey ->
+ Decrypted =
+ ?aes_gcm_decrypt(
+ DecryptionKey,
+ <<0:96>>,
+ <<UUID/binary, 0:8, Key/binary>>,
+ CipherText,
+ <<CipherTag:128>>),
+ if Decrypted /= error -> Decrypted; true ->
+ erlang:error(decryption_failed)
+ end
+ end;
+ _ ->
+ erlang:error(not_ciphertext)
+ end.
+
+
+%% cache functions
+
+insert(Cache, UUID, DbKey) ->
+ Entry = #entry{uuid = UUID, encryption_key = DbKey},
+ true = ets:insert(Cache, Entry),
+ true = ets:insert(?KEY_CHECK, {UUID, true}),
+ ok.
+
+
+lookup(Cache, UUID) ->
+ case ets:lookup(Cache, UUID) of
+ [#entry{uuid = UUID, encryption_key = DbKey}] ->
+ {ok, DbKey};
+ [] ->
+ {error, not_found}
+ end.
diff --git a/src/fabric/src/fabric.app.src b/src/aegis/src/aegis_sup.erl
similarity index 52%
copy from src/fabric/src/fabric.app.src
copy to src/aegis/src/aegis_sup.erl
index 0538b19..6d3ee83 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/aegis/src/aegis_sup.erl
@@ -10,24 +10,37 @@
% License for the specific language governing permissions and limitations under
% the License.
-{application, fabric, [
- {description, "Routing and proxying layer for CouchDB cluster"},
- {vsn, git},
- {mod, {fabric2_app, []}},
- {registered, [
- fabric_server
- ]},
- {applications, [
- kernel,
- stdlib,
- config,
- couch_epi,
- couch,
- ctrace,
- rexi,
- mem3,
- couch_log,
- couch_stats,
- erlfdb
- ]}
-]}.
+-module(aegis_sup).
+
+-behaviour(supervisor).
+
+-vsn(1).
+
+
+-export([
+ start_link/0
+]).
+
+-export([
+ init/1
+]).
+
+
+start_link() ->
+ supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+
+init([]) ->
+ Flags = #{
+ strategy => one_for_one,
+ intensity => 5,
+ period => 10
+ },
+ Children = [
+ #{
+ id => aegis_server,
+ start => {aegis_server, start_link, []},
+ shutdown => 5000
+ }
+ ],
+ {ok, {Flags, Children}}.
diff --git a/src/fabric/src/fabric.app.src b/src/aegis/test/aegis_basic_test.erl
similarity index 57%
copy from src/fabric/src/fabric.app.src
copy to src/aegis/test/aegis_basic_test.erl
index 0538b19..61d9737 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/aegis/test/aegis_basic_test.erl
@@ -10,24 +10,8 @@
% License for the specific language governing permissions and limitations under
% the License.
-{application, fabric, [
- {description, "Routing and proxying layer for CouchDB cluster"},
- {vsn, git},
- {mod, {fabric2_app, []}},
- {registered, [
- fabric_server
- ]},
- {applications, [
- kernel,
- stdlib,
- config,
- couch_epi,
- couch,
- ctrace,
- rexi,
- mem3,
- couch_log,
- couch_stats,
- erlfdb
- ]}
-]}.
+-module(aegis_basic_test).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-define(DB, #{uuid => <<"foo">>}).
diff --git a/src/aegis/test/aegis_server_test.erl b/src/aegis/test/aegis_server_test.erl
new file mode 100644
index 0000000..0f23a3f
--- /dev/null
+++ b/src/aegis/test/aegis_server_test.erl
@@ -0,0 +1,165 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(aegis_server_test).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+-define(DB, #{uuid => <<0:64>>}).
+-define(VALUE, <<0:8>>).
+-define(ENCRYPTED, <<1,155,242,89,190,54,112,151,18,145,25,251,217,
+ 49,147,125,14,162,146,201,189,100,232,38,239,111,163,84,25,60,
+ 147,167,237,107,24,204,171,232,227,16,72,203,101,118,150,252,
+ 204,80,245,66,98,213,223,63,111,105,101,154>>).
+-define(TIMEOUT, 10000).
+
+
+
+basic_test_() ->
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ {"init_db returns true when encryption enabled",
+ {timeout, ?TIMEOUT, fun test_init_db/0}},
+ {"open_db returns true when encryption enabled",
+ {timeout, ?TIMEOUT, fun test_open_db/0}},
+ {"init_db caches key",
+ {timeout, ?TIMEOUT, fun test_init_db_cache/0}},
+ {"open_db caches key",
+ {timeout, ?TIMEOUT, fun test_open_db_cache/0}},
+ {"encrypt fetches and caches key when it's missing",
+ {timeout, ?TIMEOUT, fun test_encrypt_cache/0}},
+ {"decrypt fetches and caches key when it's missing",
+ {timeout, ?TIMEOUT, fun test_decrypt_cache/0}}
+ ]
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([fabric]),
+ meck:new([?AEGIS_KEY_MANAGER], [passthrough]),
+ ok = meck:expect(?AEGIS_KEY_MANAGER, init_db, 2, {ok, <<0:256>>}),
+ ok = meck:expect(?AEGIS_KEY_MANAGER, open_db, 1, {ok, <<0:256>>}),
+ Ctx.
+
+
+teardown(Ctx) ->
+ meck:unload(),
+ test_util:stop_couch(Ctx).
+
+
+test_init_db() ->
+ ?assert(aegis_server:init_db(?DB, [])),
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)).
+
+
+test_open_db() ->
+ ?assert(aegis_server:open_db(?DB)),
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)).
+
+
+test_init_db_cache() ->
+ ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)),
+
+ ?assert(aegis_server:init_db(?DB, [])),
+
+ lists:foreach(fun(I) ->
+ Encrypted = aegis_server:encrypt(?DB, <<I:64>>, ?VALUE),
+ ?assertNotEqual(?VALUE, Encrypted),
+ ?assertMatch(<<1:8, _/binary>>, Encrypted)
+ end, lists:seq(1, 12)),
+
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)).
+
+
+test_open_db_cache() ->
+ ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)),
+
+ ?assert(aegis_server:open_db(?DB)),
+
+ lists:foreach(fun(I) ->
+ Encrypted = aegis_server:encrypt(?DB, <<I:64>>, ?VALUE),
+ ?assertNotEqual(?VALUE, Encrypted),
+ ?assertMatch(<<1:8, _/binary>>, Encrypted)
+ end, lists:seq(1, 12)),
+
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)).
+
+
+test_encrypt_cache() ->
+ ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)),
+
+ Encrypted = aegis_server:encrypt(?DB, <<1:64>>, ?VALUE),
+ ?assertNotEqual(?VALUE, Encrypted),
+ ?assertMatch(<<1:8, _/binary>>, Encrypted),
+
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)).
+
+
+test_decrypt_cache() ->
+ ?assertEqual(0, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)),
+
+ Decrypted = aegis_server:decrypt(?DB, <<1:64>>, ?ENCRYPTED),
+ ?assertEqual(<<0>>, Decrypted),
+
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)).
+
+
+
+disabled_test_() ->
+ {
+ foreach,
+ fun() ->
+ Ctx = setup(),
+ ok = meck:delete(?AEGIS_KEY_MANAGER, init_db, 2),
+ ok = meck:expect(?AEGIS_KEY_MANAGER, init_db, 2, false),
+ ok = meck:delete(?AEGIS_KEY_MANAGER, open_db, 1),
+ ok = meck:expect(?AEGIS_KEY_MANAGER, open_db, 1, false),
+ Ctx
+ end,
+ fun teardown/1,
+ [
+ {"init_db returns false when encryptions disabled",
+ {timeout, ?TIMEOUT, fun test_disabled_init_db/0}},
+ {"open_db returns false when encryptions disabled",
+ {timeout, ?TIMEOUT, fun test_disabled_open_db/0}},
+ {"pass through on encrypt when encryption disabled",
+ {timeout, ?TIMEOUT, fun test_disabled_encrypt/0}},
+ {"pass through on decrypt when encryption disabled",
+ {timeout, ?TIMEOUT, fun test_disabled_decrypt/0}}
+ ]
+ }.
+
+
+test_disabled_init_db() ->
+ ?assertNot(aegis_server:init_db(?DB, [])),
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, init_db, 2)).
+
+
+test_disabled_open_db() ->
+ ?assertNot(aegis_server:open_db(?DB)),
+ ?assertEqual(1, meck:num_calls(?AEGIS_KEY_MANAGER, open_db, 1)).
+
+
+test_disabled_encrypt() ->
+ Db = ?DB#{is_encrypted => aegis_server:open_db(?DB)},
+ Encrypted = aegis:encrypt(Db, <<1:64>>, ?VALUE),
+ ?assertEqual(?VALUE, Encrypted).
+
+
+test_disabled_decrypt() ->
+ Db = ?DB#{is_encrypted => aegis_server:open_db(?DB)},
+ Decrypted = aegis:decrypt(Db, <<1:64>>, ?ENCRYPTED),
+ ?assertEqual(?ENCRYPTED, Decrypted).
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 4640258..699601c 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -357,6 +357,10 @@ catch_error(HttpReq, throw, Error) ->
send_error(HttpReq, Error);
catch_error(HttpReq, error, database_does_not_exist) ->
send_error(HttpReq, database_does_not_exist);
+catch_error(HttpReq, error, decryption_failed) ->
+ send_error(HttpReq, decryption_failed);
+catch_error(HttpReq, error, not_ciphertext) ->
+ send_error(HttpReq, not_ciphertext);
catch_error(HttpReq, Tag, Error) ->
Stack = erlang:get_stacktrace(),
% TODO improve logging and metrics collection for client disconnects
@@ -965,6 +969,10 @@ error_info(not_implemented) ->
error_info(timeout) ->
{500, <<"timeout">>, <<"The request could not be processed in a reasonable"
" amount of time.">>};
+error_info(decryption_failed) ->
+ {500, <<"decryption_failed">>, <<"Decryption failed">>};
+error_info(not_ciphertext) ->
+ {500, <<"not_ciphertext">>, <<"Not Ciphertext">>};
error_info({service_unavailable, Reason}) ->
{503, <<"service unavailable">>, Reason};
error_info({timeout, _Reason}) ->
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 2181e53..c957222 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -161,7 +161,7 @@ fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) ->
callback => Callback,
acc => Acc0
},
- Fun = fun fold_fwd/2,
+ Fun = aegis:wrap_fold_fun(TxDb, fun fold_fwd/2),
#{
acc := Acc1
@@ -321,7 +321,7 @@ update_id_idx(TxDb, Sig, ViewId, DocId, NewRows, KVSize) ->
Key = id_idx_key(DbPrefix, Sig, DocId, ViewId),
Val = couch_views_encoding:encode([length(NewRows), KVSize, Unique]),
- ok = erlfdb:set(Tx, Key, Val).
+ ok = erlfdb:set(Tx, Key, aegis:encrypt(TxDb, Key, Val)).
update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) ->
@@ -341,7 +341,7 @@ update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) ->
lists:foreach(fun({DupeId, Key1, Key2, EV}) ->
KK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId),
Val = erlfdb_tuple:pack({Key2, EV}),
- ok = erlfdb:set(Tx, KK, Val)
+ ok = erlfdb:set(Tx, KK, aegis:encrypt(TxDb, KK, Val))
end, KVsToAdd).
@@ -356,7 +356,7 @@ get_view_keys(TxDb, Sig, DocId) ->
erlfdb_tuple:unpack(K, DbPrefix),
[TotalKeys, TotalSize, UniqueKeys] = couch_views_encoding:decode(V),
{ViewId, TotalKeys, TotalSize, UniqueKeys}
- end, erlfdb:get_range(Tx, Start, End, [])).
+ end, aegis:decrypt(TxDb, erlfdb:get_range(Tx, Start, End, []))).
update_row_count(TxDb, Sig, ViewId, Increment) ->
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index bf3e2aa..234c529 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -42,6 +42,7 @@
-define(DB_LOCAL_DOC_BODIES, 25).
-define(DB_ATT_NAMES, 26).
-define(DB_SEARCH, 27).
+-define(DB_AEGIS, 28).
% Versions
diff --git a/src/fabric/src/fabric.app.src b/src/fabric/src/fabric.app.src
index 0538b19..a7059fd 100644
--- a/src/fabric/src/fabric.app.src
+++ b/src/fabric/src/fabric.app.src
@@ -28,6 +28,7 @@
mem3,
couch_log,
couch_stats,
- erlfdb
+ erlfdb,
+ aegis
]}
]}.
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index b1ada52..ba57e64 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -181,7 +181,7 @@ create(#{} = Db0, Options) ->
name := DbName,
tx := Tx,
layer_prefix := LayerPrefix
- } = Db = ensure_current(Db0, false),
+ } = Db1 = ensure_current(Db0, false),
DbKey = erlfdb_tuple:pack({?ALL_DBS, DbName}, LayerPrefix),
HCA = erlfdb_hca:create(erlfdb_tuple:pack({?DB_HCA}, LayerPrefix)),
@@ -224,7 +224,7 @@ create(#{} = Db0, Options) ->
UserCtx = fabric2_util:get_value(user_ctx, Options, #user_ctx{}),
Options1 = lists:keydelete(user_ctx, 1, Options),
- Db#{
+ Db2 = Db1#{
uuid => UUID,
db_prefix => DbPrefix,
db_version => DbVersion,
@@ -240,7 +240,8 @@ create(#{} = Db0, Options) ->
% All other db things as we add features,
db_options => Options1
- }.
+ },
+ aegis:init_db(Db2, Options).
open(#{} = Db0, Options) ->
@@ -286,14 +287,15 @@ open(#{} = Db0, Options) ->
},
Db3 = load_config(Db2),
+ Db4 = aegis:open_db(Db3),
- case {UUID, Db3} of
+ case {UUID, Db4} of
{undefined, _} -> ok;
{<<_/binary>>, #{uuid := UUID}} -> ok;
{<<_/binary>>, #{uuid := _}} -> erlang:error(database_does_not_exist)
end,
- load_validate_doc_funs(Db3).
+ load_validate_doc_funs(Db4).
% Match on `name` in the function head since some non-fabric2 db
@@ -701,9 +703,10 @@ get_doc_body_wait(#{} = Db0, DocId, RevInfo, Future) ->
rev_path := RevPath
} = RevInfo,
- RevBodyRows = erlfdb:fold_range_wait(Tx, Future, fun({_K, V}, Acc) ->
+ FoldFun = aegis:wrap_fold_fun(Db, fun({_K, V}, Acc) ->
[V | Acc]
- end, []),
+ end),
+ RevBodyRows = erlfdb:fold_range_wait(Tx, Future, FoldFun, []),
BodyRows = lists:reverse(RevBodyRows),
fdb_to_doc(Db, DocId, RevPos, [Rev | RevPath], BodyRows).
@@ -720,7 +723,7 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) ->
Prefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, DocId}, DbPrefix),
Future = erlfdb:get_range_startswith(Tx, Prefix),
- Chunks = lists:map(fun({_K, V}) -> V end, erlfdb:wait(Future)),
+ {_, Chunks} = lists:unzip(aegis:decrypt(Db, erlfdb:wait(Future))),
fdb_to_local_doc(Db, DocId, Rev, Chunks).
@@ -949,7 +952,9 @@ write_local_doc(#{} = Db0, Doc) ->
% Make sure to clear the whole range, in case there was a larger
% document body there before.
erlfdb:clear_range_startswith(Tx, BPrefix),
- lists:foreach(fun({K, V}) -> erlfdb:set(Tx, K, V) end, Rows)
+ lists:foreach(fun({K, V}) ->
+ erlfdb:set(Tx, K, aegis:encrypt(Db, K, V))
+ end, Rows)
end,
case {WasDeleted, Doc#doc.deleted} of
@@ -977,8 +982,8 @@ read_attachment(#{} = Db, DocId, AttId) ->
not_found ->
throw({not_found, missing});
KVs ->
- Vs = [V || {_K, V} <- KVs],
- iolist_to_binary(Vs)
+ {_, Chunks} = lists:unzip(aegis:decrypt(Db, KVs)),
+ iolist_to_binary(Chunks)
end,
IdKey = erlfdb_tuple:pack({?DB_ATT_NAMES, DocId, AttId}, DbPrefix),
@@ -1023,7 +1028,7 @@ write_attachment(#{} = Db, DocId, Data, Encoding)
lists:foldl(fun(Chunk, ChunkId) ->
AttKey = erlfdb_tuple:pack({?DB_ATTS, DocId, AttId, ChunkId}, DbPrefix),
- ok = erlfdb:set(Tx, AttKey, Chunk),
+ ok = erlfdb:set(Tx, AttKey, aegis:encrypt(Db, AttKey, Chunk)),
ChunkId + 1
end, 0, Chunks),
{ok, AttId}.
@@ -1332,7 +1337,7 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
Rows = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
- ok = erlfdb:set(Tx, Key, Value)
+ ok = erlfdb:set(Tx, Key, aegis:encrypt(Db, Key, Value))
end, Rows).