You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2022/06/17 14:17:31 UTC
[couchdb] branch database_encryption updated (4cb37bffe -> a70ca0c63)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch database_encryption
in repository https://gitbox.apache.org/repos/asf/couchdb.git
discard 4cb37bffe Database Encryption Support
new a70ca0c63 Database Encryption Support
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4cb37bffe)
\
N -- N -- N refs/heads/database_encryption (a70ca0c63)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch/src/couch_file.erl | 32 +++++++++++++++++++++-----------
1 file changed, 21 insertions(+), 11 deletions(-)
[couchdb] 01/01: Database Encryption Support
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch database_encryption
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a70ca0c63e0b7f7d780a0539888f81ed0f7a9c57
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu May 5 14:39:37 2022 +0100
Database Encryption Support
CouchDB can optionally encrypt databases and views.
We use AES in Counter Mode, which ensures we can encrypt and decrypt
any section of the file without padding or alignment. The ciphertext
is the same length as the plaintext. This mode provides
confidentiality but not authentication.
Key management is configurable, a system administrator can write a
module implementing the aegis_key_manager behaviour with any
implementation. CouchDB ships with an implementation that stores keys
in the config file as an example, but this is not suitable for
production.
---
src/couch/src/couch_bt_engine.erl | 7 +-
src/couch/src/couch_bt_engine_compactor.erl | 13 +-
src/couch/src/couch_db_updater.erl | 3 +-
src/couch/src/couch_encryption_manager.erl | 56 ++++
src/couch/src/couch_encryption_manager_demo.erl | 46 +++
src/couch/src/couch_file.erl | 383 +++++++++++++++++++++---
src/couch_mrview/src/couch_mrview_compactor.erl | 2 +-
src/couch_mrview/src/couch_mrview_index.erl | 2 +-
src/couch_mrview/src/couch_mrview_util.erl | 8 +-
9 files changed, 466 insertions(+), 54 deletions(-)
diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index 486ed7cb0..c7dcf0d0d 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -827,14 +827,15 @@ copy_props(#st{header = Header} = St, Props) ->
needs_commit = true
}}.
-open_db_file(FilePath, Options) ->
- case couch_file:open(FilePath, Options) of
+open_db_file(FilePath, Options0) ->
+ case couch_file:open(FilePath, Options0) of
{ok, Fd} ->
{ok, Fd};
{error, enoent} ->
% Couldn't find file. is there a compact version? This ca
% happen (rarely) if we crashed during the file switch.
- case couch_file:open(FilePath ++ ".compact", [nologifmissing]) of
+ Options1 = couch_encryption_manager:encryption_options(Options0),
+ case couch_file:open(FilePath ++ ".compact", [nologifmissing | Options1]) of
{ok, Fd} ->
Fmt = "Recovering from compaction file: ~s~s",
couch_log:info(Fmt, [FilePath, ".compact"]),
diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl
index 8ed55b5c3..ea0bde962 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -54,7 +54,7 @@ start(#st{} = St, DbName, Options, Parent) ->
couch_db_engine:trigger_on_compact(DbName),
?COMP_EVENT(init),
- {ok, InitCompSt} = open_compaction_files(DbName, St, Options),
+ {ok, InitCompSt} = open_compaction_files(DbName, St, [{db_name, DbName} | Options]),
?COMP_EVENT(files_opened),
Stages = [
@@ -94,8 +94,9 @@ open_compaction_files(DbName, OldSt, Options) ->
} = OldSt,
DataFile = DbFilePath ++ ".compact.data",
MetaFile = DbFilePath ++ ".compact.meta",
- {ok, DataFd, DataHdr} = open_compaction_file(DataFile),
- {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile),
+ EncryptionOptions = couch_encryption_manager:encryption_options(Options),
+ {ok, DataFd, DataHdr} = open_compaction_file(DataFile, EncryptionOptions),
+ {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile, EncryptionOptions),
DataHdrIsDbHdr = couch_bt_engine_header:is_header(DataHdr),
CompSt =
case {DataHdr, MetaHdr} of
@@ -623,15 +624,15 @@ compact_final_sync(#comp_st{new_st = St0} = CompSt) ->
new_st = St1
}.
-open_compaction_file(FilePath) ->
- case couch_file:open(FilePath, [nologifmissing]) of
+open_compaction_file(FilePath, FileOpenOptions) ->
+ case couch_file:open(FilePath, [nologifmissing | FileOpenOptions]) of
{ok, Fd} ->
case couch_file:read_header(Fd) of
{ok, Header} -> {ok, Fd, Header};
no_valid_header -> {ok, Fd, nil}
end;
{error, enoent} ->
- {ok, Fd} = couch_file:open(FilePath, [create]),
+ {ok, Fd} = couch_file:open(FilePath, [create | FileOpenOptions]),
{ok, Fd, nil}
end.
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index 17a1e9160..ffe8d6057 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -37,7 +37,8 @@ init({Engine, DbName, FilePath, Options0}) ->
erlang:put(io_priority, {db_update, DbName}),
update_idle_limit_from_config(),
DefaultSecObj = default_security_object(DbName),
- Options = [{default_security_object, DefaultSecObj} | Options0],
+ Options =
+ [{db_name, DbName}, {default_security_object, DefaultSecObj} | Options0],
try
{ok, EngineState} = couch_db_engine:init(Engine, FilePath, Options),
Db = init_db(DbName, FilePath, EngineState, Options),
diff --git a/src/couch/src/couch_encryption_manager.erl b/src/couch/src/couch_encryption_manager.erl
new file mode 100644
index 000000000..962ca868a
--- /dev/null
+++ b/src/couch/src/couch_encryption_manager.erl
@@ -0,0 +1,56 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_encryption_manager).
+
+-export([new_dek/1, unwrap_dek/2, encryption_options/1]).
+
+-callback new_dek(DbName :: binary()) ->
+ {ok, KeyID :: binary(), DEK :: binary(), WEK :: binary()}
+ | dont_encrypt
+ | {error, Reason :: term()}.
+
+-callback unwrap_dek(KeyID :: binary(), WEK :: binary()) ->
+ {ok, DEK :: binary()}
+ | {ok, NewKeyID :: binary(), DEK :: binary(), NewWEK :: binary()}
+ | {error, Reason :: term()}.
+
+new_dek(DbName) ->
+ case manager() of
+ undefined ->
+ dont_encrypt;
+ Module ->
+ Module:new_dek(DbName)
+ end.
+
+unwrap_dek(KeyID, WEK) ->
+ case manager() of
+ undefined ->
+ {error, encryption_not_supported};
+ Manager ->
+ Manager:unwrap_dek(KeyID, WEK)
+ end.
+
+manager() ->
+ case config:get("encryption", "manager") of
+ undefined ->
+ undefined;
+ Module ->
+ list_to_atom(Module)
+ end.
+
+%% Extract just the encryption related options from an options list.
+encryption_options(Options) ->
+ case lists:keyfind(db_name, 1, Options) of
+ false -> [];
+ {db_name, DbName} -> [{db_name, DbName}]
+ end.
diff --git a/src/couch/src/couch_encryption_manager_demo.erl b/src/couch/src/couch_encryption_manager_demo.erl
new file mode 100644
index 000000000..ed70425ec
--- /dev/null
+++ b/src/couch/src/couch_encryption_manager_demo.erl
@@ -0,0 +1,46 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_encryption_manager_demo).
+-behaviour(couch_encryption_manager).
+
+-export([new_dek/1, unwrap_dek/2]).
+
+new_dek(_DbName) ->
+ KeyID = <<"default">>,
+ KEK = <<0:256>>,
+ DEK = crypto:strong_rand_bytes(32),
+ {ok, KeyID, DEK, wrap_key(KeyID, KEK, DEK)}.
+
+unwrap_dek(<<"default">> = KeyID, WEK) ->
+ KEK = <<0:256>>,
+ unwrap_key(KeyID, KEK, WEK);
+unwrap_dek(_, _) ->
+ {error, invalid_key_id}.
+
+wrap_key(KeyID, KEK, DEK) when is_binary(KEK), is_binary(DEK) ->
+ IV = crypto:strong_rand_bytes(16),
+ {<<_:32/binary>> = CipherText, <<_:16/binary>> = CipherTag} =
+ crypto:crypto_one_time_aead(aes_256_gcm, KEK, IV, DEK, KeyID, 16, true),
+ <<IV:16/binary, CipherText/binary, CipherTag/binary>>.
+
+unwrap_key(KeyID, KEK, <<IV:16/binary, CipherText:32/binary, CipherTag:16/binary>>) when
+ is_binary(KEK)
+->
+ case crypto:crypto_one_time_aead(aes_256_gcm, KEK, IV, CipherText, KeyID, CipherTag, false) of
+ error ->
+ {error, unwrap_failed};
+ DEK ->
+ {ok, DEK}
+ end;
+unwrap_key(_KeyID, _KEK, _) ->
+ {error, malformed_wrapped_key}.
\ No newline at end of file
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
index ba8d9c42f..50fd24951 100644
--- a/src/couch/src/couch_file.erl
+++ b/src/couch/src/couch_file.erl
@@ -20,9 +20,35 @@
-define(MONITOR_CHECK, 10000).
% 4 KiB
-define(SIZE_BLOCK, 16#1000).
+-define(ENCRYPTION_HEADER_SIZE, 16#800).
-define(IS_OLD_STATE(S), is_pid(S#file.db_monitor)).
-define(PREFIX_SIZE, 5).
-define(DEFAULT_READ_COUNT, 1024).
+-define(ENCRYPTED_HEADER_MARKER, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).
+
+%% Database encryption design details
+
+%% On file creation, if an encryption manager is configured, the
+%% manager is asked to generate a new data encryption key (DEK). The
+%% manager either returns a new DEK along with a wrapped form of the
+%% same value that is safe to store in plain view (WEK) or it returns
+%% `dont_encrypt` to indicate that the database should not be
+%% encrypted.
+
+%% When opening an encrypted file the encryption manager is asked to
+%% unwrap the WEK retrieved from encryption header. The manager can
+%% return just the unwrapped DEK or can return the unwrapped DEK and a
+%% new WEK (wrapping the same DEK value) and a new KeyID. This
+%% supports efficient key rotation.
+
+%% The first 4096 bytes of an encrypted file contains two copies of
+%% the encrypted header, to protect against partial writes. If the two
+%% copies are valid but different the first copy is used.
+
+%% All bytes, excepting the first 4096 bytes, are encrypted with AES
+%% in Counter Mode. A random initialisation vector is generated at
+%% file creation and stored in the encryption header. This
+%% initialisation vector is re-randomised if the file is truncated.
-type block_id() :: non_neg_integer().
-type location() :: non_neg_integer().
@@ -33,9 +59,38 @@
is_sys,
eof = 0,
db_monitor,
- pread_limit = 0
+ pread_limit = 0,
+ iv,
+ dek
}).
+-define(encrypt_ctr(File, Pos, Data),
+ crypto:stream_encrypt(
+ crypto:stream_init(aes_ctr, File#file.dek, aes_ctr(File#file.iv, Pos)), Data
+ )
+).
+-define(decrypt_ctr(File, Pos, Data),
+ crypto:stream_decrypt(
+ crypto:stream_init(aes_ctr, File#file.dek, aes_ctr(File#file.iv, Pos)), Data
+ )
+).
+
+-ifdef(OTP_RELEASE).
+-if(?OTP_RELEASE >= 22).
+
+-undef(encrypt_ctr).
+-define(encrypt_ctr(File, Pos, Data),
+ crypto:crypto_one_time(aes_256_ctr, File#file.dek, aes_ctr(File#file.iv, Pos), Data, true)
+).
+
+-undef(decrypt_ctr).
+-define(decrypt_ctr(File, Pos, Data),
+ crypto:crypto_one_time(aes_256_ctr, File#file.dek, aes_ctr(File#file.iv, Pos), Data, false)
+).
+
+-endif.
+-endif.
+
% public API
-export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]).
-export([pread_term/2, pread_iolist/2, pread_binary/2]).
@@ -439,7 +494,20 @@ init({Filepath, Options, ReturnPid, Ref}) ->
ok = file:sync(Fd),
maybe_track_open_os_files(Options),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}};
+ case
+ init_crypto(
+ Filepath,
+ #file{
+ fd = Fd, is_sys = IsSys, pread_limit = Limit
+ },
+ Options
+ )
+ of
+ {ok, File} ->
+ {ok, File};
+ Error ->
+ init_status_error(ReturnPid, Ref, Error)
+ end;
false ->
ok = file:close(Fd),
init_status_error(ReturnPid, Ref, {error, eexist})
@@ -447,7 +515,20 @@ init({Filepath, Options, ReturnPid, Ref}) ->
false ->
maybe_track_open_os_files(Options),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}}
+ case
+ init_crypto(
+ Filepath,
+ #file{
+ fd = Fd, is_sys = IsSys, pread_limit = Limit
+ },
+ Options
+ )
+ of
+ {ok, File} ->
+ {ok, File};
+ Error ->
+ init_status_error(ReturnPid, Ref, Error)
+ end
end;
Error ->
init_status_error(ReturnPid, Ref, Error)
@@ -464,7 +545,20 @@ init({Filepath, Options, ReturnPid, Ref}) ->
maybe_track_open_os_files(Options),
{ok, Eof} = file:position(Fd, eof),
erlang:send_after(?INITIAL_WAIT, self(), maybe_close),
- {ok, #file{fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit}};
+ case
+ init_crypto(
+ Filepath,
+ #file{
+ fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit
+ },
+ Options
+ )
+ of
+ {ok, File} ->
+ {ok, File};
+ Error ->
+ init_status_error(ReturnPid, Ref, Error)
+ end;
Error ->
init_status_error(ReturnPid, Ref, Error)
end;
@@ -563,7 +657,9 @@ handle_call(sync, _From, #file{fd = Fd} = File) ->
% can't fathom.
{stop, Error, Error, #file{fd = nil}}
end;
-handle_call({truncate, Pos}, _From, #file{fd = Fd} = File) ->
+handle_call({truncate, Pos}, _From, #file{fd = Fd, dek = DEK} = File) when
+ DEK == undefined orelse Pos >= ?SIZE_BLOCK
+->
{ok, Pos} = file:position(Fd, Pos),
case file:truncate(Fd) of
ok ->
@@ -571,16 +667,41 @@ handle_call({truncate, Pos}, _From, #file{fd = Fd} = File) ->
Error ->
{reply, Error, File}
end;
-handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
+%% truncating an encrypted file earlier than the end of the encryption header.
+%% reuse the wrapped key with a new iv.
+handle_call({truncate, Pos}, _From, #file{fd = Fd, dek = DEK} = File0) when
+ DEK /= undefined andalso Pos < ?SIZE_BLOCK
+->
+ case read_encryption_header(File0) of
+ {ok, {KeyID, WEK, _IV}} ->
+ {ok, 0} = file:position(Fd, 0),
+ case file:truncate(Fd) of
+ ok ->
+ File1 = File0#file{eof = 0},
+ IV = new_aes_iv(),
+ case write_encryption_header(File1, KeyID, WEK, IV) of
+ {ok, File2} ->
+ ok = file:sync(File2#file.fd),
+ {reply, ok, init_crypto_file(File2, DEK, IV)};
+ {error, Reason} ->
+ {reply, {error, Reason}, File1}
+ end;
+ Error ->
+ {reply, Error, File0}
+ end;
+ {error, Reason} ->
+ {reply, {error, Reason}, File0}
+ end;
+handle_call({append_bin, Bin}, _From, #file{eof = Pos} = File) ->
Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin),
Size = iolist_size(Blocks),
- case file:write(Fd, Blocks) of
+ case encrypted_write(File, Blocks) of
ok ->
{reply, {ok, Pos, Size}, File#file{eof = Pos + Size}};
Error ->
{reply, Error, reset_eof(File)}
end;
-handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) ->
+handle_call({append_bins, Bins}, _From, #file{eof = Pos} = File) ->
{BlockResps, FinalPos} = lists:mapfoldl(
fun(Bin, PosAcc) ->
Blocks = make_blocks(PosAcc rem ?SIZE_BLOCK, Bin),
@@ -591,13 +712,13 @@ handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) ->
Bins
),
{AllBlocks, Resps} = lists:unzip(BlockResps),
- case file:write(Fd, AllBlocks) of
+ case encrypted_write(File, AllBlocks) of
ok ->
{reply, {ok, Resps}, File#file{eof = FinalPos}};
Error ->
{reply, Error, reset_eof(File)}
end;
-handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
+handle_call({write_header, Bin}, _From, #file{eof = Pos} = File) ->
BinSize = byte_size(Bin),
case Pos rem ?SIZE_BLOCK of
0 ->
@@ -606,14 +727,14 @@ handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
Padding = <<0:(8 * (?SIZE_BLOCK - BlockOffset))>>
end,
FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])],
- case file:write(Fd, FinalBin) of
+ case encrypted_write(File, FinalBin) of
ok ->
{reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}};
Error ->
{reply, Error, reset_eof(File)}
end;
-handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) ->
- {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}.
+handle_call(find_header, _From, #file{eof = Pos} = File) ->
+ {reply, find_header(File, Pos div ?SIZE_BLOCK), File}.
handle_cast(close, Fd) ->
{stop, normal, Fd}.
@@ -641,26 +762,26 @@ format_status(_Opt, [PDict, #file{} = File]) ->
{_Fd, FilePath} = couch_util:get_value(couch_file_fd, PDict),
[{data, [{"State", File}, {"InitialFilePath", FilePath}]}].
-find_header(Fd, Block) ->
- case (catch load_header(Fd, Block)) of
+find_header(#file{} = File, Block) ->
+ case (catch load_header(File, Block)) of
{ok, Bin} ->
{ok, Bin};
_Error ->
ReadCount = config:get_integer(
"couchdb", "find_header_read_count", ?DEFAULT_READ_COUNT
),
- find_header(Fd, Block - 1, ReadCount)
+ find_header(File, Block - 1, ReadCount)
end.
-load_header(Fd, Block) ->
+load_header(#file{} = File, Block) ->
{ok, <<1, HeaderLen:32/integer, RestBlock/binary>>} =
- file:pread(Fd, Block * ?SIZE_BLOCK, ?SIZE_BLOCK),
- load_header(Fd, Block * ?SIZE_BLOCK, HeaderLen, RestBlock).
+ encrypted_pread(File, Block * ?SIZE_BLOCK, ?SIZE_BLOCK),
+ load_header(File, Block * ?SIZE_BLOCK, HeaderLen, RestBlock).
-load_header(Fd, Pos, HeaderLen) ->
- load_header(Fd, Pos, HeaderLen, <<>>).
+load_header(#file{} = File, Pos, HeaderLen) ->
+ load_header(File, Pos, HeaderLen, <<>>).
-load_header(Fd, Pos, HeaderLen, RestBlock) ->
+load_header(#file{} = File, Pos, HeaderLen, RestBlock) ->
TotalBytes = calculate_total_read_len(?PREFIX_SIZE, HeaderLen),
RawBin =
case TotalBytes =< byte_size(RestBlock) of
@@ -670,7 +791,7 @@ load_header(Fd, Pos, HeaderLen, RestBlock) ->
false ->
ReadStart = Pos + ?PREFIX_SIZE + byte_size(RestBlock),
ReadLen = TotalBytes - byte_size(RestBlock),
- {ok, Missing} = file:pread(Fd, ReadStart, ReadLen),
+ {ok, Missing} = encrypted_pread(File, ReadStart, ReadLen),
<<RestBlock/binary, Missing/binary>>
end,
<<Md5Sig:16/binary, HeaderBin/binary>> =
@@ -681,12 +802,12 @@ load_header(Fd, Pos, HeaderLen, RestBlock) ->
%% Read multiple block locations using a single file:pread/2.
-spec find_header(file:fd(), block_id(), non_neg_integer()) ->
{ok, binary()} | no_valid_header.
-find_header(_Fd, Block, _ReadCount) when Block < 0 ->
+find_header(_File, Block, _ReadCount) when Block < 0 ->
no_valid_header;
-find_header(Fd, Block, ReadCount) ->
+find_header(#file{} = File, Block, ReadCount) ->
FirstBlock = max(0, Block - ReadCount + 1),
BlockLocations = [?SIZE_BLOCK * B || B <- lists:seq(FirstBlock, Block)],
- {ok, DataL} = file:pread(Fd, [{L, ?PREFIX_SIZE} || L <- BlockLocations]),
+ {ok, DataL} = encrypted_pread(File, [{L, ?PREFIX_SIZE} || L <- BlockLocations]),
%% Since BlockLocations are ordered from oldest to newest, we rely
%% on lists:foldl/3 to reverse the order, making HeaderLocations
%% correctly ordered from newest to oldest.
@@ -700,27 +821,27 @@ find_header(Fd, Block, ReadCount) ->
[],
lists:zip(BlockLocations, DataL)
),
- case find_newest_header(Fd, HeaderLocations) of
+ case find_newest_header(File, HeaderLocations) of
{ok, _Location, HeaderBin} ->
{ok, HeaderBin};
_ ->
ok = file:advise(
- Fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need
+ File#file.fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need
),
NextBlock = hd(BlockLocations) div ?SIZE_BLOCK - 1,
- find_header(Fd, NextBlock, ReadCount)
+ find_header(File, NextBlock, ReadCount)
end.
-spec find_newest_header(file:fd(), [{location(), header_size()}]) ->
{ok, location(), binary()} | not_found.
-find_newest_header(_Fd, []) ->
+find_newest_header(_File, []) ->
not_found;
-find_newest_header(Fd, [{Location, Size} | LocationSizes]) ->
- case (catch load_header(Fd, Location, Size)) of
+find_newest_header(#file{} = File, [{Location, Size} | LocationSizes]) ->
+ case (catch load_header(File, Location, Size)) of
{ok, HeaderBin} ->
{ok, Location, HeaderBin};
_Error ->
- find_newest_header(Fd, LocationSizes)
+ find_newest_header(File, LocationSizes)
end.
-spec read_raw_iolist_int(#file{}, Pos :: non_neg_integer(), Len :: non_neg_integer()) ->
@@ -728,9 +849,9 @@ find_newest_header(Fd, [{Location, Size} | LocationSizes]) ->
% 0110 UPGRADE CODE
read_raw_iolist_int(Fd, {Pos, _Size}, Len) ->
read_raw_iolist_int(Fd, Pos, Len);
-read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) ->
+read_raw_iolist_int(#file{} = File, Pos, Len) ->
{Pos, TotalBytes} = get_pread_locnum(File, Pos, Len),
- case catch file:pread(Fd, Pos, TotalBytes) of
+ case catch encrypted_pread(File, Pos, TotalBytes) of
{ok, <<RawBin:TotalBytes/binary>>} ->
{remove_block_prefixes(Pos rem ?SIZE_BLOCK, RawBin), Pos + TotalBytes};
Else ->
@@ -744,15 +865,15 @@ read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) ->
throw({file_truncate_error, Else, Filepath})
end.
-% TODO: check if this is really unused
-read_multi_raw_iolists_int(#file{fd = Fd} = File, PosLens) ->
+% used in couch_bt_engine_compactor.erl via pread_terms/2
+read_multi_raw_iolists_int(#file{} = File, PosLens) ->
LocNums = lists:map(
fun({Pos, Len}) ->
get_pread_locnum(File, Pos, Len)
end,
PosLens
),
- {ok, Bins} = file:pread(Fd, LocNums),
+ {ok, Bins} = encrypted_pread(File, LocNums),
lists:zipwith(
fun({Pos, TotalBytes}, Bin) ->
<<RawBin:TotalBytes/binary>> = Bin,
@@ -905,6 +1026,192 @@ reset_eof(#file{} = File) ->
{ok, Eof} = file:position(File#file.fd, eof),
File#file{eof = Eof}.
+%% new file.
+init_crypto(_Filepath, #file{eof = 0, dek = undefined} = File0, Options) ->
+ case lists:keyfind(db_name, 1, Options) of
+ {db_name, DbName} ->
+ case couch_encryption_manager:new_dek(DbName) of
+ {ok, KeyID, DEK, WEK} ->
+ IV = new_aes_iv(),
+ case write_encryption_header(File0, KeyID, WEK, IV) of
+ {ok, File1} ->
+ ok = file:sync(File1#file.fd),
+ {ok, init_crypto_file(File1, DEK, IV)};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ dont_encrypt ->
+ {ok, File0};
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ false ->
+ {ok, File0}
+ end;
+%% we're opening an existing file and need to unwrap the key if file is encrypted.
+init_crypto(Filepath, #file{eof = Eof, dek = undefined} = File, _Options) when Eof >= ?SIZE_BLOCK ->
+ case read_encryption_header(File) of
+ {ok, {KeyID, WEK, IV}} ->
+ case couch_encryption_manager:unwrap_dek(KeyID, WEK) of
+ {ok, DEK} ->
+ {ok, init_crypto_file(File, DEK, IV)};
+ {ok, NewKeyID, DEK, NewWEK} ->
+ %% manager has rewrapped the DEK with a new key, update our header.
+ case file:open(Filepath, [read, write, raw]) of
+ {ok, Fd} ->
+ case write_encryption_header(#file{fd = Fd}, NewKeyID, NewWEK, IV) of
+ ok ->
+ ok = file:sync(Fd),
+ ok = file:close(Fd),
+ {ok, init_crypto_file(File, DEK, IV)};
+ {error, Reason} ->
+ ok = file:close(Fd),
+ {error, Reason}
+ end;
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ {error, Reason} ->
+ {error, Reason}
+ end;
+ not_encrypted ->
+ {ok, File};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+init_crypto_file(#file{} = File, DEK, IV) when is_binary(DEK), is_binary(IV) ->
+ File#file{iv = crypto:bytes_to_integer(IV), dek = DEK}.
+
+write_encryption_header(#file{eof = 0} = File, KeyID, WrappedKey, IV) when
+ bit_size(KeyID) =< 128, bit_size(WrappedKey) == 512, bit_size(IV) == 128
+->
+ Header = [<<?ENCRYPTED_HEADER_MARKER>>, <<(byte_size(KeyID)):16>>, KeyID, IV, WrappedKey],
+ PaddedHeader = [Header, <<0:((?ENCRYPTION_HEADER_SIZE - iolist_size(Header) - 32) * 8)>>],
+ DigestHeader = [PaddedHeader, crypto:hash(sha256, PaddedHeader)],
+ ?ENCRYPTION_HEADER_SIZE = iolist_size(DigestHeader),
+ case file:write(File#file.fd, [DigestHeader, DigestHeader]) of
+ ok ->
+ {ok, File#file{eof = ?SIZE_BLOCK}};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+read_encryption_header(#file{} = File) ->
+ case file:pread(File#file.fd, 0, ?SIZE_BLOCK) of
+ {ok,
+ <<?ENCRYPTED_HEADER_MARKER, Bin1:(?ENCRYPTION_HEADER_SIZE - 16)/binary,
+ ?ENCRYPTED_HEADER_MARKER, Bin2/binary>>} ->
+ Header1 = extract_header(<<?ENCRYPTED_HEADER_MARKER, Bin1/binary>>),
+ Header2 = extract_header(<<?ENCRYPTED_HEADER_MARKER, Bin2/binary>>),
+ {_Fd, Filepath} = get(couch_file_fd),
+ case {Header1, Header2} of
+ {{ok, Header}, {ok, Header}} ->
+ % headers match
+ {ok, Header};
+ {{ok, Header1}, {ok, Header2}} ->
+ couch_log:warning(
+ "~s: Encryption header version differences.~nPrimary header: ~p~nSecondary header: ~p. Using primary header",
+ [Filepath, Header1, Header2]
+ ),
+ {ok, Header1};
+ {{ok, Header}, {error, Reason}} ->
+ couch_log:warning(
+ "~s: Secondary encryption header corruption (error: ~p). Using primary header.",
+ [
+ Filepath, Reason
+ ]
+ ),
+ {ok, Header};
+ {{error, Reason}, {ok, Header}} ->
+ couch_log:warning(
+ "~s: Primary encryption header corruption (error: ~p). Using secondary header.",
+ [
+ Filepath, Reason
+ ]
+ ),
+ {ok, Header};
+ {{error, Reason1}, {error, Reason2}} ->
+ couch_log:error("~s: Both encryption headers corrupted (errors: ~p, ~p).", [
+ Filepath, Reason1, Reason2
+ ]),
+ {error, corrupted_encryption_header}
+ end;
+ {ok, _} ->
+ not_encrypted;
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+extract_header(
+ <<?ENCRYPTED_HEADER_MARKER, KeyIDLen:16, KeyID:(KeyIDLen)/binary, IV:16/binary,
+ WrappedKey:64/binary, _/binary>> = Bin
+) ->
+ case check_header(Bin) of
+ true ->
+ {ok, {KeyID, WrappedKey, IV}};
+ false ->
+ {error, corrupted_encryption_header}
+ end;
+extract_header(_) ->
+ {error, corrupted_encryption_header}.
+
+check_header(Bin) ->
+ Data = binary:part(Bin, 0, byte_size(Bin) - 32),
+ Digest = binary:part(Bin, byte_size(Bin), -32),
+ Digest == crypto:hash(sha256, Data).
+
+%% We can encrypt any section of the file but we must make
+%% sure we align with the key stream.
+encrypted_write(#file{dek = undefined} = File, Data) ->
+ file:write(File#file.fd, Data);
+encrypted_write(#file{} = File, Data) ->
+ CipherText = ?encrypt_ctr(File, File#file.eof, pad(File#file.eof, Data)),
+ file:write(File#file.fd, unpad(File#file.eof, CipherText)).
+
+encrypted_pread(#file{dek = undefined} = File, LocNums) ->
+ file:pread(File#file.fd, LocNums);
+encrypted_pread(#file{} = File, LocNums) ->
+ case file:pread(File#file.fd, LocNums) of
+ {ok, DataL} ->
+ {ok,
+ lists:zipwith(
+ fun({Pos, _Len}, CipherText) ->
+ PlainText = ?decrypt_ctr(File, Pos, pad(Pos, CipherText)),
+ unpad(Pos, PlainText)
+ end,
+ LocNums,
+ DataL
+ )};
+ Else ->
+ Else
+ end.
+
+encrypted_pread(#file{dek = undefined} = File, Pos, Len) ->
+ file:pread(File#file.fd, Pos, Len);
+encrypted_pread(#file{} = File, Pos, Len) ->
+ case file:pread(File#file.fd, Pos, Len) of
+ {ok, CipherText} ->
+ PlainText = ?decrypt_ctr(File, Pos, pad(Pos, CipherText)),
+ {ok, unpad(Pos, PlainText)};
+ Else ->
+ Else
+ end.
+
+new_aes_iv() ->
+ crypto:strong_rand_bytes(16).
+
+aes_ctr(IV, Pos) ->
+ <<(IV + (Pos div 16)):128>>.
+
+pad(Pos, IOData) ->
+ [<<0:(Pos rem 16 * 8)>>, IOData].
+
+unpad(Pos, Bin) when is_binary(Bin) ->
+ Size = Pos rem 16 * 8,
+ <<_:Size, Result/binary>> = Bin,
+ Result.
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl
index 28e5a9b3d..5471b124b 100644
--- a/src/couch_mrview/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview/src/couch_mrview_compactor.erl
@@ -47,7 +47,7 @@ compact(State) ->
{EmptyState, NumDocIds} = couch_util:with_db(DbName, fun(Db) ->
CompactFName = couch_mrview_util:compaction_file(DbName, Sig),
- {ok, Fd} = couch_mrview_util:open_file(CompactFName),
+ {ok, Fd} = couch_mrview_util:open_file(CompactFName, [{db_name, DbName}]),
ESt = couch_mrview_util:reset_index(Db, Fd, State),
{ok, Count} = couch_db:get_doc_count(Db),
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 1bfdb2818..b4528c27d 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -120,7 +120,7 @@ open(Db, State0) ->
OldSig = couch_mrview_util:maybe_update_index_file(State),
- case couch_mrview_util:open_file(IndexFName) of
+ case couch_mrview_util:open_file(IndexFName, [{db_name, DbName}]) of
{ok, Fd} ->
case couch_file:read_header(Fd) of
% upgrade code for <= 2.x
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 9e3d292ed..f9c533376 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -17,7 +17,7 @@
-export([verify_view_filename/1, get_signature_from_filename/1]).
-export([ddoc_to_mrst/2, init_state/4, reset_index/3]).
-export([make_header/1]).
--export([index_file/2, compaction_file/2, open_file/1]).
+-export([index_file/2, compaction_file/2, open_file/2]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
@@ -791,10 +791,10 @@ compaction_file(DbName, Sig) ->
FileName = couch_index_util:hexsig(Sig) ++ ".compact.view",
couch_index_util:index_file(mrview, DbName, FileName).
-open_file(FName) ->
- case couch_file:open(FName, [nologifmissing]) of
+open_file(FName, Options) ->
+ case couch_file:open(FName, [nologifmissing | Options]) of
{ok, Fd} -> {ok, Fd};
- {error, enoent} -> couch_file:open(FName, [create]);
+ {error, enoent} -> couch_file:open(FName, [create | Options]);
Error -> Error
end.