You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2022/05/06 16:01:35 UTC
[couchdb] 01/02: demonstrate native encryption
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch aegis_3.x
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 18e8f8cdc1eb9eb714985926b570f92b96684b79
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu May 5 14:39:37 2022 +0100
demonstrate native encryption
Key management is _not_ done and so this scheme is not secure.
The AES_MASTER_KEY value should be retrieved interactively and need
not be the same for different databases.
Overwriting the first 40 bytes of the file with any other value
renders the file unreadable.
We use AES in Counter Mode, which ensures we can encrypt and decrypt
any section of the file without padding or alignment. The ciphertext
is the same length as the plaintext. This mode provides
confidentiality but not authentication.
---
src/couch/src/couch_file.erl | 115 +++++++++++++++++++++++++++++++++++-----
src/couch/src/couch_keywrap.erl | 115 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 216 insertions(+), 14 deletions(-)
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
index 0e786525f..350cdc75c 100644
--- a/src/couch/src/couch_file.erl
+++ b/src/couch/src/couch_file.erl
@@ -33,7 +33,8 @@
is_sys,
eof = 0,
db_monitor,
- pread_limit = 0
+ pread_limit = 0,
+ key
}).
% public API
@@ -62,6 +63,8 @@
%% or {error, Reason} if the file could not be opened.
%%----------------------------------------------------------------------
+-define(AES_MASTER_KEY, <<0:256>>).
+
open(Filepath) ->
open(Filepath, []).
@@ -453,7 +456,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
ok = file:sync(Fd),
maybe_track_open_os_files(Options),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}};
+ init_key(#file{fd = Fd, is_sys = IsSys, pread_limit = Limit});
false ->
ok = file:close(Fd),
init_status_error(ReturnPid, Ref, {error, eexist})
@@ -461,7 +464,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
false ->
maybe_track_open_os_files(Options),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}}
+ init_key(#file{fd = Fd, is_sys = IsSys, pread_limit = Limit})
end;
Error ->
init_status_error(ReturnPid, Ref, Error)
@@ -478,7 +481,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
maybe_track_open_os_files(Options),
{ok, Eof} = file:position(Fd, eof),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit}};
+ init_key(#file{fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit});
Error ->
init_status_error(ReturnPid, Ref, Error)
end;
@@ -581,20 +584,25 @@ handle_call({truncate, Pos}, _From, #file{fd = Fd} = File) ->
{ok, Pos} = file:position(Fd, Pos),
case file:truncate(Fd) of
ok ->
- {reply, ok, File#file{eof = Pos}};
+ case init_key(File#file{eof = Pos}) of
+ {ok, File1} ->
+ {reply, ok, File1};
+ {error, Reason} ->
+ {error, Reason}
+ end;
Error ->
{reply, Error, File}
end;
-handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
+handle_call({append_bin, Bin}, _From, #file{eof = Pos} = File) ->
Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin),
Size = iolist_size(Blocks),
- case file:write(Fd, Blocks) of
+ case encrypted_write(File, Blocks) of
ok ->
{reply, {ok, Pos, Size}, File#file{eof = Pos + Size}};
Error ->
{reply, Error, reset_eof(File)}
end;
-handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) ->
+handle_call({append_bins, Bins}, _From, #file{eof = Pos} = File) ->
{BlockResps, FinalPos} = lists:mapfoldl(
fun(Bin, PosAcc) ->
Blocks = make_blocks(PosAcc rem ?SIZE_BLOCK, Bin),
@@ -605,7 +613,7 @@ handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) ->
Bins
),
{AllBlocks, Resps} = lists:unzip(BlockResps),
- case file:write(Fd, AllBlocks) of
+ case encrypted_write(File, AllBlocks) of
ok ->
{reply, {ok, Resps}, File#file{eof = FinalPos}};
Error ->
@@ -742,9 +750,9 @@ find_newest_header(Fd, [{Location, Size} | LocationSizes]) ->
% 0110 UPGRADE CODE
read_raw_iolist_int(Fd, {Pos, _Size}, Len) ->
read_raw_iolist_int(Fd, Pos, Len);
-read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) ->
+read_raw_iolist_int(#file{} = File, Pos, Len) ->
{Pos, TotalBytes} = get_pread_locnum(File, Pos, Len),
- case catch file:pread(Fd, Pos, TotalBytes) of
+ case catch encrypted_pread(File, Pos, TotalBytes) of
{ok, <<RawBin:TotalBytes/binary>>} ->
{remove_block_prefixes(Pos rem ?SIZE_BLOCK, RawBin), Pos + TotalBytes};
Else ->
@@ -758,15 +766,15 @@ read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) ->
throw({file_truncate_error, Else, Filepath})
end.
-% TODO: check if this is really unused
-read_multi_raw_iolists_int(#file{fd = Fd} = File, PosLens) ->
+% used in couch_bt_engine_compactor.erl via pread_terms/2
+read_multi_raw_iolists_int(#file{} = File, PosLens) ->
LocNums = lists:map(
fun({Pos, Len}) ->
get_pread_locnum(File, Pos, Len)
end,
PosLens
),
- {ok, Bins} = file:pread(Fd, LocNums),
+ {ok, Bins} = encrypted_pread(File, LocNums),
lists:zipwith(
fun({Pos, TotalBytes}, Bin) ->
<<RawBin:TotalBytes/binary>> = Bin,
@@ -919,6 +927,85 @@ reset_eof(#file{} = File) ->
{ok, Eof} = file:position(File#file.fd, eof),
File#file{eof = Eof}.
+
+%% we've wiped all the data, including the wrapped key, so we need a new one.
+init_key(#file{eof = 0} = File) ->
+ Key = crypto:strong_rand_bytes(32),
+ WrappedKey = couch_keywrap:key_wrap(?AES_MASTER_KEY, Key),
+ ok = file:write(File#file.fd, WrappedKey),
+ ok = file:sync(File#file.fd),
+ {ok, File#file{eof = iolist_size(WrappedKey), key = Key}};
+
+%% we're opening an existing file and need to unwrap the key.
+init_key(#file{key = undefined} = File) ->
+ {ok, WrappedKey} = file:pread(File#file.fd, 0, 40),
+ case couch_keywrap:key_unwrap(?AES_MASTER_KEY, WrappedKey) of
+ fail ->
+ {error, cannot_unwrap_key};
+ Key when is_binary(Key) ->
+ {ok, File#file{key = Key}}
+ end;
+
+%% we're opening an existing file that contains a wrapped key
+%% which we've already unwrapped.
+init_key(#file{eof = Eof, key = Key} = File) when Eof > 40, is_binary(Key) ->
+ {ok, File}.
+
+
+%% We can encrypt any section of the file but we must make
+%% sure we align with the key stream.
+encrypted_write(#file{} = File, Data) ->
+ CipherText = encrypt(File#file.key, File#file.eof, pad(File#file.eof, Data)),
+ file:write(File#file.fd, unpad(File#file.eof, CipherText)).
+
+
+encrypted_pread(#file{} = File, LocNums) ->
+ case file:pread(File#file.fd, LocNums) of
+ {ok, DataL} ->
+ {ok, lists:zipwith(
+ fun({Pos, _Len}, CipherText) ->
+ PlainText = decrypt(File#file.key, Pos, pad(Pos, CipherText)),
+ unpad(Pos, PlainText) end,
+ LocNums,
+ DataL)};
+ Else ->
+ Else
+ end.
+
+
+encrypted_pread(#file{} = File, Pos, Len) ->
+ case file:pread(File#file.fd, Pos, Len) of
+ {ok, CipherText} ->
+ PlainText = decrypt(File#file.key, Pos, pad(Pos, CipherText)),
+ {ok, unpad(Pos, PlainText)};
+ Else ->
+ Else
+ end.
+
+
+encrypt(Key, Pos, Data) ->
+ IV = aes_ctr_iv(Pos),
+ crypto:crypto_one_time(aes_256_ctr, Key, IV, Data, true).
+
+
+decrypt(Key, Pos, Data) ->
+ IV = aes_ctr_iv(Pos),
+ crypto:crypto_one_time(aes_256_ctr, Key, IV, Data, false).
+
+
+aes_ctr_iv(Pos) ->
+ <<(Pos div 16):128>>.
+
+
+pad(Pos, IOData) ->
+ [<<0:(Pos rem 16 * 8)>>, IOData].
+
+
+unpad(Pos, Bin) when is_binary(Bin) ->
+ <<_:(Pos rem 16 * 8), Result/binary>> = Bin,
+ Result.
+
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/couch/src/couch_keywrap.erl b/src/couch/src/couch_keywrap.erl
new file mode 100644
index 000000000..2cfa2b104
--- /dev/null
+++ b/src/couch/src/couch_keywrap.erl
@@ -0,0 +1,115 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_keywrap).
+
+%% Implementation of NIST Special Publication 800-38F
+%% For wrapping and unwrapping keys with AES.
+
+-export([key_wrap/2, key_unwrap/2]).
+
+-define(ICV1, 16#A6A6A6A6A6A6A6A6).
+
+-spec key_wrap(WrappingKey :: binary(), KeyToWrap :: binary()) -> binary().
+key_wrap(WrappingKey, KeyToWrap) when
+ is_binary(WrappingKey), bit_size(KeyToWrap) rem 64 == 0
+->
+ N = bit_size(KeyToWrap) div 64,
+ wrap(WrappingKey, <<?ICV1:64>>, KeyToWrap, 1, 6 * N).
+
+wrap(_WrappingKey, A, R, T, End) when T > End ->
+ <<A/binary, R/binary>>;
+wrap(WrappingKey, A, R, T, End) ->
+ <<R1:64, Rest/binary>> = R,
+ <<MSB_B:64, LSB_B:64>> = crypto:crypto_one_time(aes_256_ecb, WrappingKey, <<A/binary, R1:64>>, true),
+ wrap(WrappingKey, <<(MSB_B bxor T):64>>, <<Rest/binary, LSB_B:64>>, T + 1, End).
+
+-spec key_unwrap(WrappingKey :: binary(), KeyToUnwrap :: binary()) -> binary() | fail.
+key_unwrap(WrappingKey, KeyToUnwrap) when
+ is_binary(WrappingKey), bit_size(KeyToUnwrap) rem 64 == 0
+->
+ N = (bit_size(KeyToUnwrap) div 64),
+ <<A:64, R/binary>> = KeyToUnwrap,
+ case unwrap(WrappingKey, <<A:64>>, R, 6 * (N - 1)) of
+ <<?ICV1:64, UnwrappedKey/binary>> ->
+ UnwrappedKey;
+ _ ->
+ fail
+ end.
+
+unwrap(_WrappingKey, A, R, 0) ->
+ <<A/binary, R/binary>>;
+unwrap(WrappingKey, <<A:64>>, R, T) ->
+ RestSize = bit_size(R) - 64,
+ <<Rest:RestSize, R2:64>> = R,
+ <<MSB_B:64, LSB_B:64>> = crypto:crypto_one_time(aes_256_ecb, WrappingKey, <<(A bxor T):64, R2:64>>, false),
+ unwrap(WrappingKey, <<MSB_B:64>>, <<LSB_B:64, Rest:RestSize>>, T - 1).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+wrap_test_() ->
+ [
+ %% 128 KEK / 128 DATA
+ test_wrap_unwrap(
+ <<16#000102030405060708090A0B0C0D0E0F:128>>,
+ <<16#00112233445566778899AABBCCDDEEFF:128>>,
+ <<16#1FA68B0A8112B447AEF34BD8FB5A7B829D3E862371D2CFE5:192>>
+ ),
+ %% 192 KEK / 128 DATA
+ test_wrap_unwrap(
+ <<16#000102030405060708090A0B0C0D0E0F1011121314151617:192>>,
+ <<16#00112233445566778899AABBCCDDEEFF:128>>,
+ <<16#96778B25AE6CA435F92B5B97C050AED2468AB8A17AD84E5D:192>>
+ ),
+ %% 256 KEK / 128 DATA
+ test_wrap_unwrap(
+ <<16#000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F:256>>,
+ <<16#00112233445566778899AABBCCDDEEFF:128>>,
+ <<16#64E8C3F9CE0F5BA263E9777905818A2A93C8191E7D6E8AE7:192>>
+ ),
+ %% 192 KEK / 192 DATA
+ test_wrap_unwrap(
+ <<16#000102030405060708090A0B0C0D0E0F1011121314151617:192>>,
+ <<16#00112233445566778899AABBCCDDEEFF0001020304050607:192>>,
+ <<16#031D33264E15D33268F24EC260743EDCE1C6C7DDEE725A936BA814915C6762D2:256>>
+ ),
+ %% 256 KEK / 192 DATA
+ test_wrap_unwrap(
+ <<16#000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F:256>>,
+ <<16#00112233445566778899AABBCCDDEEFF0001020304050607:192>>,
+ <<16#A8F9BC1612C68B3FF6E6F4FBE30E71E4769C8B80A32CB8958CD5D17D6B254DA1:256>>
+ ),
+ %% 256 KEK / 256 DATA
+ test_wrap_unwrap(
+ <<16#000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F:256>>,
+ <<16#00112233445566778899AABBCCDDEEFF000102030405060708090A0B0C0D0E0F:256>>,
+ <<
+ 16#28C9F404C4B810F4CBCCB35CFB87F8263F5786E2D80ED326CBC7F0E71A99F43BFB988B9B7A02DD21:320
+ >>
+ )
+ ].
+
+test_wrap_unwrap(WrappingKey, KeyToWrap, ExpectedWrappedKey) ->
+ [
+ ?_assertEqual(ExpectedWrappedKey, key_wrap(WrappingKey, KeyToWrap)),
+ ?_assertEqual(KeyToWrap, key_unwrap(WrappingKey, key_wrap(WrappingKey, KeyToWrap)))
+ ].
+
+fail_test() ->
+ KEK = <<16#000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F:256>>,
+ CipherText = <<
+ 16#28C9F404C4B810F4CBCCB35CFB87F8263F5786E2D80ED326CBC7F0E71A99F43BFB988B9B7A02DD20:320
+ >>,
+ ?assertEqual(fail, key_unwrap(KEK, CipherText)).
+
+-endif.