Re: [PR] feat(encryption): add key manager [incubator-pegasus]

["acelyc111 (via GitHub)" <gi...@apache.org>]

acelyc111 commented on code in PR #1706:
URL: https://github.com/apache/incubator-pegasus/pull/1706#discussion_r1440420888


##########
src/replica/replica_stub.cpp:
##########
@@ -389,9 +441,41 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f
         }
     }
 
+    std::string server_key;
+    dsn::replication::replica_kms_info kms_info;
+    if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) {
+        key_provider.reset(new dsn::security::KMSKeyProvider(
+            ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()),
+            FLAGS_encryption_cluster_key_name));
+        auto error_code = kms_info.load(_options.data_dirs[0]);
+        if (error_code != dsn::ERR_OK) {
+            LOG_WARNING("It's normal to encounter a temporary inability to open the kms-info file "
+                        "during the first process launch. error_code = {}",
+                        error_code);
+        }
+        // Upon the first launch, the encryption key should be empty. The process will then retrieve
+        // EEK, IV, and KV from KMS.
+        // After the first launch, the encryption key, obtained from the kms-info file, should not
+        // be empty. The process will then acquire the DEK from KMS.
+        std::string kms_path =
+            utils::filesystem::path_combine(_options.data_dirs[0], replica_kms_info::kKmsInfo);
+        if (!utils::filesystem::path_exists(kms_path)) {
+            auto err = key_provider->GenerateEncryptionKey(&kms_info);
+            CHECK(err, "get encryption key failed, err = {}", err);
+        }
+        auto err = key_provider->DecryptEncryptionKey(kms_info, &server_key);
+        CHECK(err, "get decryption key failed, err = {}", err);
+        FLAGS_server_key = server_key.c_str();
+    }
+
     // Initialize the file system manager.
     _fs_manager.initialize(_options.data_dirs, _options.data_dir_tags);
 
+    if (key_provider) {
+        auto err = kms_info.store(_options.data_dirs[0]);
+        CHECK(err == dsn::ERR_OK, "Can't store kms key to kms-info file, err = {}", err);

Review Comment:
   ```suggestion
           CHECK_EQ_MSG(dsn::ERR_OK, err, "Can't store kms key to kms-info file, err = {}", err);
   ```



##########
src/runtime/security/kms_client.cpp:
##########
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <initializer_list>
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "absl/strings/escaping.h"
+#include "fmt/core.h"
+#include "http/http_client.h"
+#include "http/http_method.h"
+#include "http/http_status_code.h"
+#include "nlohmann/json.hpp"
+#include "nlohmann/json_fwd.hpp"
+#include "runtime/security/kms_client.h"
+#include "runtime/security/replica_kms_info.h"
+#include "utils/error_code.h"
+#include "utils/fmt_logging.h"
+
+namespace dsn {
+namespace security {
+
+dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info,
+                                             std::string *decrypted_key)
+{
+    nlohmann::json payload;
+    payload["name"] = cluster_key_name_;
+    std::string iv_plain = ::absl::HexStringToBytes(kms_info.iv);
+    std::string iv_b64;
+    ::absl::WebSafeBase64Escape(iv_plain, &iv_b64);
+    payload["iv"] = iv_b64;
+    std::string eek_plain = ::absl::HexStringToBytes(kms_info.eek);
+    std::string eek_b64;
+    ::absl::WebSafeBase64Escape(eek_plain, &eek_b64);
+    payload["material"] = eek_b64;
+
+    http_client client;
+    auto err = client.init();
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed");
+    }
+
+    err = client.set_auth(http_auth_type::SPNEGO);
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed");
+    }
+
+    std::vector<std::string> urls;
+    urls.reserve(kms_urls_.size());
+    for (const auto &url : kms_urls_) {
+        urls.emplace_back(fmt::format("{}/v1/keyversion/{}/_eek?eek_op=decrypt", url, kms_info.kv));
+    }
+
+    client.clear_header_fields();
+    client.set_content_type("application/json");
+    client.set_accept("*/*");
+    err = client.with_post_method(payload.dump());
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "http client set method failed");
+    }
+
+    nlohmann::json j;
+    for (const auto &url : urls) {
+        err = client.set_url(url);
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http clientt set url failed");
+        }
+        std::string resp;
+        err = client.exec_method(&resp);
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http client exec post method failed");
+        }
+        http_status_code http_status;
+        client.get_http_status(http_status);
+        if (http_status == http_status_code::kOk) {
+            j = nlohmann::json::parse(resp);
+            break;
+        } else {
+            LOG_WARNING("The http status is ({}), and url is ({})",
+                        get_http_status_message(http_status),
+                        url);
+        }
+    }
+
+    std::string dek_b64;
+    if (j.contains("material")) {
+        dek_b64 = j.at("material");
+    } else {
+        return dsn::error_s::make(ERR_INVALID_DATA, "Null material received");
+    }
+    std::string dek_plain;
+    if (!::absl::WebSafeBase64Unescape(dek_b64, &dek_plain)) {
+        return dsn::error_s::make(ERR_INVALID_DATA, "Invalid IV received");
+    }
+    *decrypted_key = ::absl::BytesToHexString(dek_plain);
+    return dsn::error_s::ok();
+}
+
+dsn::error_s KMSClient::GenerateEncryptionKeyFromKMS(const std::string &key_name,
+                                                     dsn::replication::replica_kms_info *kms_info)
+{
+    http_client client;
+    auto err = client.init();
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed");
+    }
+    err = client.set_auth(http_auth_type::SPNEGO);
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed");
+    }
+    std::vector<std::string> urls;
+    urls.reserve(kms_urls_.size());
+    for (const auto &url : kms_urls_) {
+        urls.emplace_back(
+            fmt::format("{}/v1/key/{}/_eek?eek_op=generate&num_keys=1", url, key_name));
+    }
+
+    nlohmann::json j = nlohmann::json::object();
+    for (const auto &url : urls) {
+        err = client.set_url(url);
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http client set url failed");
+        }
+
+        err = client.with_get_method();
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http client set get method failed");
+        }
+
+        std::string resp;
+        err = client.exec_method(&resp);
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http client exec get method failed");
+        }
+
+        http_status_code http_status;
+        client.get_http_status(http_status);
+        if (http_status == http_status_code::kOk) {
+            j = nlohmann::json::parse(resp);
+            nlohmann::json jsonObject = j.at(0);
+            std::string res = jsonObject.dump();
+            j = nlohmann::json::parse(res);
+            break;
+        } else {
+            LOG_WARNING("The http status is ({}), and url is ({})",
+                        get_http_status_message(http_status),
+                        url);
+        }

Review Comment:
   ```suggestion
           }
   
           LOG_WARNING("The http status is ({}), and url is ({})",
                       get_http_status_message(http_status),
                       url);
   ```



##########
src/replica/replica_stub.cpp:
##########
@@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f
         }
     }
 
+    std::string server_key;
+    dsn::replication::replica_kms_info kms_info;
+    if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) {

Review Comment:
   Didn't update this?



##########
src/runtime/security/kms_client.cpp:
##########
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <initializer_list>
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "absl/strings/escaping.h"
+#include "fmt/core.h"
+#include "http/http_client.h"
+#include "http/http_method.h"
+#include "http/http_status_code.h"
+#include "nlohmann/json.hpp"
+#include "nlohmann/json_fwd.hpp"
+#include "runtime/security/kms_client.h"
+#include "runtime/security/replica_kms_info.h"
+#include "utils/error_code.h"
+#include "utils/fmt_logging.h"
+
+namespace dsn {
+namespace security {
+
+dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info,
+                                             std::string *decrypted_key)
+{
+    nlohmann::json payload;
+    payload["name"] = cluster_key_name_;
+    std::string iv_plain = ::absl::HexStringToBytes(kms_info.iv);
+    std::string iv_b64;
+    ::absl::WebSafeBase64Escape(iv_plain, &iv_b64);
+    payload["iv"] = iv_b64;
+    std::string eek_plain = ::absl::HexStringToBytes(kms_info.eek);
+    std::string eek_b64;
+    ::absl::WebSafeBase64Escape(eek_plain, &eek_b64);
+    payload["material"] = eek_b64;
+
+    http_client client;
+    auto err = client.init();
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed");
+    }
+
+    err = client.set_auth(http_auth_type::SPNEGO);
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed");
+    }
+
+    std::vector<std::string> urls;
+    urls.reserve(kms_urls_.size());
+    for (const auto &url : kms_urls_) {
+        urls.emplace_back(fmt::format("{}/v1/keyversion/{}/_eek?eek_op=decrypt", url, kms_info.kv));
+    }
+
+    client.clear_header_fields();
+    client.set_content_type("application/json");
+    client.set_accept("*/*");
+    err = client.with_post_method(payload.dump());
+    if (!err.is_ok()) {
+        return dsn::error_s::make(ERR_CURL_FAILED, "http client set method failed");
+    }
+
+    nlohmann::json j;
+    for (const auto &url : urls) {
+        err = client.set_url(url);
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http clientt set url failed");
+        }
+        std::string resp;
+        err = client.exec_method(&resp);
+        if (!err.is_ok()) {
+            return dsn::error_s::make(ERR_CURL_FAILED, "http client exec post method failed");
+        }
+        http_status_code http_status;
+        client.get_http_status(http_status);
+        if (http_status == http_status_code::kOk) {
+            j = nlohmann::json::parse(resp);
+            break;
+        } else {
+            LOG_WARNING("The http status is ({}), and url is ({})",
+                        get_http_status_message(http_status),
+                        url);
+        }

Review Comment:
   ```suggestion
           }
           
           LOG_WARNING("The http status is ({}), and url is ({})",
                       get_http_status_message(http_status),
                       url);
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org