You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pegasus.apache.org by "empiredan (via GitHub)" <gi...@apache.org> on 2023/09/20 07:51:39 UTC
[GitHub] [incubator-pegasus] empiredan commented on a diff in pull request #1606: feat(encryption): introduce PegasusEnv
empiredan commented on code in PR #1606:
URL: https://github.com/apache/incubator-pegasus/pull/1606#discussion_r1331106258
##########
src/utils/env.cpp:
##########
@@ -0,0 +1,200 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "env.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include <fmt/core.h>
+#include <rocksdb/convenience.h>
+#include <rocksdb/env.h>
+#include <rocksdb/env_encryption.h>
+#include <rocksdb/slice.h>
+
+#include "utils/defer.h"
+#include "utils/filesystem.h"
+#include "utils/flags.h"
+#include "utils/fmt_logging.h"
+#include "utils/utils.h"
+
+DSN_DEFINE_bool(pegasus.server,
+ encrypt_data_at_rest,
+ false,
+ "Whether the sensitive files should be encrypted on the file system.");
+
+DSN_DEFINE_string(pegasus.server,
+ server_key_for_testing,
+ "server_key_for_testing",
+ "The encrypted server key to use in the filesystem. NOTE: only for testing.");
+
+DSN_DEFINE_string(pegasus.server,
+ encryption_method,
+ "AES128CTR",
+ "The encryption method to use in the filesystem. Now "
+ "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR.");
+
+namespace dsn {
+namespace utils {
+
+rocksdb::Env *NewEncryptedEnv()
+{
+ // Create an encryption provider.
+ std::shared_ptr<rocksdb::EncryptionProvider> provider;
+ std::string provider_id =
Review Comment:
```suggestion
auto provider_id =
```
##########
src/utils/env.cpp:
##########
@@ -0,0 +1,200 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "env.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include <fmt/core.h>
+#include <rocksdb/convenience.h>
+#include <rocksdb/env.h>
+#include <rocksdb/env_encryption.h>
+#include <rocksdb/slice.h>
+
+#include "utils/defer.h"
+#include "utils/filesystem.h"
+#include "utils/flags.h"
+#include "utils/fmt_logging.h"
+#include "utils/utils.h"
+
+DSN_DEFINE_bool(pegasus.server,
+ encrypt_data_at_rest,
+ false,
+ "Whether the sensitive files should be encrypted on the file system.");
+
+DSN_DEFINE_string(pegasus.server,
+ server_key_for_testing,
+ "server_key_for_testing",
+ "The encrypted server key to use in the filesystem. NOTE: only for testing.");
+
+DSN_DEFINE_string(pegasus.server,
+ encryption_method,
+ "AES128CTR",
+ "The encryption method to use in the filesystem. Now "
+ "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR.");
+
+namespace dsn {
+namespace utils {
+
+rocksdb::Env *NewEncryptedEnv()
+{
+ // Create an encryption provider.
+ std::shared_ptr<rocksdb::EncryptionProvider> provider;
+ std::string provider_id =
+ fmt::format("AES:{},{}", FLAGS_server_key_for_testing, FLAGS_encryption_method);
+ auto s = rocksdb::EncryptionProvider::CreateFromString(
+ rocksdb::ConfigOptions(), provider_id, &provider);
+ CHECK(s.ok(), "Failed to create encryption provider: {}", s.ToString());
+
+ // Create an encrypted env.
+ return NewEncryptedEnv(rocksdb::Env::Default(), provider);
+}
+
+rocksdb::Env *PegasusEnv(FileDataType type)
+{
+ // Return an encrypted env only when the file is sensitive and FLAGS_encrypt_data_at_rest
+ // is enabled at the same time.
+ if (FLAGS_encrypt_data_at_rest && type == FileDataType::kSensitive) {
+ static rocksdb::Env *env = NewEncryptedEnv();
+ return env;
+ }
+
+ // Otherwise, return a common non-encrypted env.
+ static rocksdb::Env *env = rocksdb::Env::Default();
+ return env;
+}
+
+namespace {
+rocksdb::Status do_copy_file(const std::string &src_fname,
+ dsn::utils::FileDataType src_type,
+ const std::string &dst_fname,
+ dsn::utils::FileDataType dst_type,
+ int64_t remain_size,
+ uint64_t *total_size)
+{
+ rocksdb::EnvOptions src_env_options;
+ std::unique_ptr<rocksdb::SequentialFile> src_file;
+ auto s =
+ dsn::utils::PegasusEnv(src_type)->NewSequentialFile(src_fname, &src_file, src_env_options);
+ LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for reading", src_fname);
+
+ // Limit the size of the file to be copied.
+ int64_t src_file_size;
+ CHECK(dsn::utils::filesystem::file_size(src_fname, src_type, src_file_size), "");
+ if (remain_size == -1) {
+ // Copy the whole file if 'remain_size' is -1.
+ remain_size = src_file_size;
+ }
+ remain_size = std::min(remain_size, src_file_size);
Review Comment:
```suggestion
} else {
remain_size = std::min(remain_size, src_file_size);
}
```
##########
src/utils/env.cpp:
##########
@@ -0,0 +1,200 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "env.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include <fmt/core.h>
+#include <rocksdb/convenience.h>
+#include <rocksdb/env.h>
+#include <rocksdb/env_encryption.h>
+#include <rocksdb/slice.h>
+
+#include "utils/defer.h"
+#include "utils/filesystem.h"
+#include "utils/flags.h"
+#include "utils/fmt_logging.h"
+#include "utils/utils.h"
+
+DSN_DEFINE_bool(pegasus.server,
+ encrypt_data_at_rest,
+ false,
+ "Whether the sensitive files should be encrypted on the file system.");
+
+DSN_DEFINE_string(pegasus.server,
+ server_key_for_testing,
+ "server_key_for_testing",
+ "The encrypted server key to use in the filesystem. NOTE: only for testing.");
+
+DSN_DEFINE_string(pegasus.server,
+ encryption_method,
+ "AES128CTR",
+ "The encryption method to use in the filesystem. Now "
+ "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR.");
+
+namespace dsn {
+namespace utils {
+
+rocksdb::Env *NewEncryptedEnv()
+{
+ // Create an encryption provider.
+ std::shared_ptr<rocksdb::EncryptionProvider> provider;
+ std::string provider_id =
+ fmt::format("AES:{},{}", FLAGS_server_key_for_testing, FLAGS_encryption_method);
+ auto s = rocksdb::EncryptionProvider::CreateFromString(
+ rocksdb::ConfigOptions(), provider_id, &provider);
+ CHECK(s.ok(), "Failed to create encryption provider: {}", s.ToString());
+
+ // Create an encrypted env.
+ return NewEncryptedEnv(rocksdb::Env::Default(), provider);
+}
+
+rocksdb::Env *PegasusEnv(FileDataType type)
+{
+ // Return an encrypted env only when the file is sensitive and FLAGS_encrypt_data_at_rest
+ // is enabled at the same time.
+ if (FLAGS_encrypt_data_at_rest && type == FileDataType::kSensitive) {
+ static rocksdb::Env *env = NewEncryptedEnv();
+ return env;
+ }
+
+ // Otherwise, return a common non-encrypted env.
+ static rocksdb::Env *env = rocksdb::Env::Default();
+ return env;
+}
+
+namespace {
+rocksdb::Status do_copy_file(const std::string &src_fname,
+ dsn::utils::FileDataType src_type,
+ const std::string &dst_fname,
+ dsn::utils::FileDataType dst_type,
+ int64_t remain_size,
+ uint64_t *total_size)
+{
+ rocksdb::EnvOptions src_env_options;
+ std::unique_ptr<rocksdb::SequentialFile> src_file;
+ auto s =
+ dsn::utils::PegasusEnv(src_type)->NewSequentialFile(src_fname, &src_file, src_env_options);
+ LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for reading", src_fname);
+
+ // Limit the size of the file to be copied.
+ int64_t src_file_size;
+ CHECK(dsn::utils::filesystem::file_size(src_fname, src_type, src_file_size), "");
Review Comment:
```suggestion
CHECK_TRUE(dsn::utils::filesystem::file_size(src_fname, src_type, src_file_size));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org