You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@doris.apache.org by GitBox <gi...@apache.org> on 2018/11/02 07:26:32 UTC
[GitHub] imay closed pull request #264: Rewrite aes encryption #257
imay closed pull request #264: Rewrite aes encryption #257
URL: https://github.com/apache/incubator-doris/pull/264
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index fd025790..2e5d866d 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -458,7 +458,6 @@ set(DORIS_LINK_LIBS
DorisGen
Webserver
TestUtil
- AES
${WL_END_GROUP}
)
@@ -557,7 +556,6 @@ add_subdirectory(${SRC_DIR}/exprs)
add_subdirectory(${SRC_DIR}/udf)
add_subdirectory(${SRC_DIR}/runtime)
add_subdirectory(${SRC_DIR}/testutil)
-add_subdirectory(${SRC_DIR}/aes)
add_subdirectory(${SRC_DIR}/tools)
# Utility CMake function to make specifying tests and benchmarks less verbose
diff --git a/be/src/aes/CMakeLists.txt b/be/src/aes/CMakeLists.txt
deleted file mode 100644
index 4749e809..00000000
--- a/be/src/aes/CMakeLists.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/aes")
-
-# where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/aes")
-
-add_library(AES STATIC
- my_aes.cpp
- my_aes_openssl.cpp
-)
diff --git a/be/src/exprs/encryption_functions.cpp b/be/src/exprs/encryption_functions.cpp
index bad97ac5..a1e856f6 100644
--- a/be/src/exprs/encryption_functions.cpp
+++ b/be/src/exprs/encryption_functions.cpp
@@ -18,7 +18,7 @@
#include "exprs/encryption_functions.h"
#include <openssl/md5.h>
-#include "aes/my_aes.h"
+#include "util/aes_util.h"
#include "exprs/anyval_util.h"
#include "exprs/expr.h"
#include "util/debug_util.h"
@@ -42,8 +42,8 @@ StringVal EncryptionFunctions::aes_encrypt(FunctionContext* ctx,
boost::scoped_array<char> p;
p.reset(new char[cipher_len]);
- int ret_code = my_aes_encrypt((unsigned char *)src.ptr, src.len,
- (unsigned char*)p.get(), (unsigned char *)key.ptr, key.len, my_aes_128_ecb, NULL);
+ int ret_code = AesUtil::encrypt(AES_128_ECB, (unsigned char*)src.ptr, src.len,
+ (unsigned char*)key.ptr, key.len, NULL, true, (unsigned char*)p.get());
if (ret_code < 0) {
return StringVal::null();
}
@@ -60,8 +60,8 @@ StringVal EncryptionFunctions::aes_decrypt(FunctionContext* ctx,
boost::scoped_array<char> p;
p.reset(new char[cipher_len]);
- int ret_code = my_aes_decrypt((unsigned char *)src.ptr, src.len, (unsigned char*)p.get(),
- (unsigned char *)key.ptr, key.len, my_aes_128_ecb, NULL);
+ int ret_code = AesUtil::decrypt(AES_128_ECB, (unsigned char*)src.ptr, src.len,
+ (unsigned char*)key.ptr, key.len, NULL, true, (unsigned char*)p.get());
if (ret_code < 0) {
return StringVal::null();
}
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index 9be4608a..4ad07b69 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -69,6 +69,7 @@ add_library(Util STATIC
cidr.cpp
core_local.cpp
uid_util.cpp
+ aes_util.cpp
)
#ADD_BE_TEST(integer-array-test)
diff --git a/be/src/util/aes_util.cpp b/be/src/util/aes_util.cpp
new file mode 100644
index 00000000..a43e8515
--- /dev/null
+++ b/be/src/util/aes_util.cpp
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "aes_util.h"
+#include <cstring>
+#include <string>
+#include <memory>
+#include <iostream>
+
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+
+#include "exprs/base64.h"
+
+namespace palo {
+
+static const int AES_MAX_KEY_LENGTH = 256;
+
+const EVP_CIPHER* get_evp_type(const AesMode mode) {
+ switch (mode) {
+ case AES_128_ECB:
+ return EVP_aes_128_ecb();
+ case AES_128_CBC:
+ return EVP_aes_128_cbc();
+ case AES_192_ECB:
+ return EVP_aes_192_ecb();
+ case AES_192_CBC:
+ return EVP_aes_192_cbc();
+ case AES_256_ECB:
+ return EVP_aes_256_ecb();
+ case AES_256_CBC:
+ return EVP_aes_256_cbc();
+ default:
+ return NULL;
+ }
+}
+
+static uint aes_mode_key_sizes[]= {
+ 128 /* AES_128_ECB */,
+ 192 /* AES_192_ECB */,
+ 256 /* AES_256_ECB */,
+ 128 /* AES_128_CBC */,
+ 192 /* AES_192_CBC */,
+ 256 /* AES_256_CBC */
+};
+
+static void aes_create_key(const unsigned char* origin_key, uint32_t key_length,
+ uint8_t* encrypt_key, AesMode mode) {
+ const uint key_size= aes_mode_key_sizes[mode] / 8;
+ uint8_t *origin_key_end= ((uint8_t*)origin_key) + key_length; /* origin key boundary*/
+
+ uint8_t *encrypt_key_end; /* encrypt key boundary */
+ encrypt_key_end= encrypt_key + key_size;
+
+ std::memset(encrypt_key, 0, key_size); /* initialize key */
+
+ uint8_t *ptr; /* Start of the encrypt key*/
+ uint8_t *origin_ptr; /* Start of the origin key */
+ for (ptr = encrypt_key, origin_ptr = (uint8_t*)origin_key;
+ origin_ptr < origin_key_end; ptr++, origin_ptr++) {
+ if (ptr == encrypt_key_end) {
+ /* loop over origin key until we used all key */
+ ptr = encrypt_key;
+ }
+ *ptr ^= *origin_ptr;
+ }
+}
+
+static int do_encrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher,
+ const unsigned char* source, uint32_t source_length, const unsigned char* encrypt_key,
+ const unsigned char* iv, bool padding, unsigned char* encrypt, int* length_ptr) {
+ int ret = EVP_EncryptInit(aes_ctx, cipher, encrypt_key, iv);
+ if (ret == 0) {
+ return ret;
+ }
+ ret = EVP_CIPHER_CTX_set_padding(aes_ctx, padding);
+ if (ret == 0) {
+ return ret;
+ }
+ int u_len = 0;
+ ret = EVP_EncryptUpdate(aes_ctx, encrypt, &u_len, source, source_length);
+ if (ret == 0) {
+ return ret;
+ }
+ int f_len = 0;
+ ret = EVP_EncryptFinal(aes_ctx, encrypt + u_len, &f_len);
+ *length_ptr = u_len + f_len;
+ return ret;
+}
+
+int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_length,
+ const unsigned char* key, uint32_t key_length, const unsigned char* iv,
+ bool padding, unsigned char* encrypt) {
+ EVP_CIPHER_CTX aes_ctx;
+ const EVP_CIPHER* cipher = get_evp_type(mode);
+ /* The encrypt key to be used for encryption */
+ unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8];
+ aes_create_key(key, key_length, encrypt_key, mode);
+
+ if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) {
+ return AES_BAD_DATA;
+ }
+ EVP_CIPHER_CTX_init(&aes_ctx);
+ int length = 0;
+ int ret = do_encrypt(&aes_ctx, cipher, source,
+ source_length, encrypt_key, iv, padding, encrypt, &length);
+ EVP_CIPHER_CTX_cleanup(&aes_ctx);
+ if (ret == 0) {
+ ERR_clear_error();
+ return AES_BAD_DATA;
+ } else {
+ return length;
+ }
+}
+
+static int do_decrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher,
+ const unsigned char* encrypt, uint32_t encrypt_length, const unsigned char* encrypt_key,
+ const unsigned char* iv, bool padding, unsigned char* decrypt_content, int* length_ptr) {
+ int ret = EVP_DecryptInit(aes_ctx, cipher, encrypt_key, iv);
+ if (ret == 0) {
+ return ret;
+ }
+ ret = EVP_CIPHER_CTX_set_padding(aes_ctx, padding);
+ if (ret == 0) {
+ return ret;
+ }
+ int u_len = 0;
+ ret = EVP_DecryptUpdate(aes_ctx, decrypt_content, &u_len, encrypt, encrypt_length);
+ if (ret == 0) {
+ return ret;
+ }
+ int f_len = 0;
+ ret = EVP_DecryptFinal_ex(aes_ctx, decrypt_content + u_len, &f_len);
+ *length_ptr = u_len + f_len;
+ return ret;
+}
+
+int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length,
+ const unsigned char* key, uint32_t key_length, const unsigned char* iv,
+ bool padding, unsigned char* decrypt_content) {
+ EVP_CIPHER_CTX aes_ctx;
+ const EVP_CIPHER* cipher = get_evp_type(mode);
+
+ /* The encrypt key to be used for decryption */
+ unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8];
+ aes_create_key(key, key_length, encrypt_key, mode);
+
+ if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) {
+ return AES_BAD_DATA;
+ }
+
+ EVP_CIPHER_CTX_init(&aes_ctx);
+ int length = 0;
+ int ret = do_decrypt(&aes_ctx, cipher, encrypt, encrypt_length,
+ encrypt_key, iv, padding, decrypt_content, &length);
+ EVP_CIPHER_CTX_cleanup(&aes_ctx);
+ if (ret > 0) {
+ return length;
+ } else {
+ ERR_clear_error();
+ return AES_BAD_DATA;
+ }
+}
+
+}
diff --git a/be/src/util/aes_util.h b/be/src/util/aes_util.h
new file mode 100644
index 00000000..5659e0ac
--- /dev/null
+++ b/be/src/util/aes_util.h
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <stdint.h>
+
+namespace palo {
+
+enum AesMode {
+ AES_128_ECB,
+ AES_192_ECB,
+ AES_256_ECB,
+ AES_128_CBC,
+ AES_192_CBC,
+ AES_256_CBC
+};
+
+enum AesState {
+ AES_SUCCESS = 0,
+ AES_BAD_DATA = -1
+};
+
+class AesUtil {
+public:
+ static int encrypt(AesMode mode, const unsigned char* source, uint32_t source_length,
+ const unsigned char* key, uint32_t key_length, const unsigned char* iv,
+ bool padding, unsigned char* encrypt);
+
+ static int decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length,
+ const unsigned char* key, uint32_t key_length, const unsigned char* iv,
+ bool padding, unsigned char* decrypt_content);
+};
+
+}
diff --git a/be/test/util/CMakeLists.txt b/be/test/util/CMakeLists.txt
index 272dfb8a..91eda18e 100644
--- a/be/test/util/CMakeLists.txt
+++ b/be/test/util/CMakeLists.txt
@@ -36,3 +36,4 @@ ADD_BE_TEST(json_util_test)
ADD_BE_TEST(byte_buffer_test2)
ADD_BE_TEST(uid_util_test)
ADD_BE_TEST(arena_test)
+ADD_BE_TEST(aes_util_test)
diff --git a/be/test/util/aes_util_test.cpp b/be/test/util/aes_util_test.cpp
new file mode 100644
index 00000000..0ac88c41
--- /dev/null
+++ b/be/test/util/aes_util_test.cpp
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/aes_util.h"
+
+#include <string>
+#include <memory>
+#include <gtest/gtest.h>
+
+#include "exprs/base64.h"
+
+namespace palo {
+
+class AesUtilTest : public testing::Test {
+public:
+ AesUtilTest() {
+ _aes_key = "doris_aes_key";
+ }
+
+private:
+ std::string _aes_key;
+};
+
+void do_aes_test(const std::string& source, const std::string& key) {
+ int cipher_len = source.length() + 16;
+ std::unique_ptr<unsigned char> dest(new unsigned char[cipher_len]);
+ int ret_code = AesUtil::encrypt(AES_128_ECB, (unsigned char *)source.c_str(), source.length(),
+ (unsigned char *)key.c_str(), key.length(), NULL, true, dest.get());
+ ASSERT_TRUE(ret_code > 0);
+ int encrypted_length = ret_code;
+ std::unique_ptr<char> decrypted(new char[source.length()]);
+ ret_code = AesUtil::decrypt(AES_128_ECB, dest.get(), encrypted_length,
+ (unsigned char *)key.c_str(), key.length(), NULL, true, (unsigned char *)decrypted.get());
+ ASSERT_TRUE(ret_code > 0);
+ std::string decrypted_content(decrypted.get(), ret_code);
+ ASSERT_EQ(source, decrypted_content);
+}
+
+TEST_F(AesUtilTest, aes_test_basic) {
+ std::string source_1 = "hello, doris";
+ do_aes_test(source_1, _aes_key);
+ std::string source_2 = "doris test";
+ do_aes_test(source_2, _aes_key);
+}
+
+TEST_F(AesUtilTest, aes_test_by_case) {
+ std::string case_1 = "9qYx8l1601oWHEVCREAqZg=="; // base64 for encrypted "hello, doris"
+ std::string source_1 = "hello, doris";
+ std::string case_2 = "nP/db4j4yqMjXv/pItaOVA=="; // base64 for encrypted "doris test"
+ std::string source_2 = "doris test";
+
+ std::unique_ptr<char> encrypt_1(new char[case_1.length()]);
+ int length_1 = base64_decode2(case_1.c_str(), case_1.length(), encrypt_1.get());
+ std::unique_ptr<char> decrypted_1(new char[case_1.length()]);
+ int ret_code = AesUtil::decrypt(AES_128_ECB, (unsigned char *)encrypt_1.get(), length_1,
+ (unsigned char *)_aes_key.c_str(), _aes_key.length(), NULL, true, (unsigned char *)decrypted_1.get());
+ ASSERT_TRUE(ret_code > 0);
+ std::string decrypted_content_1(decrypted_1.get(), ret_code);
+ ASSERT_EQ(source_1, decrypted_content_1);
+
+ std::unique_ptr<char> encrypt_2(new char[case_2.length()]);
+ int length_2 = base64_decode2(case_2.c_str(), case_2.length(), encrypt_2.get());
+ std::unique_ptr<char> decrypted_2(new char[case_2.length()]);
+ ret_code = AesUtil::decrypt(AES_128_ECB, (unsigned char *)encrypt_2.get(), length_2,
+ (unsigned char *)_aes_key.c_str(), _aes_key.length(), NULL, true, (unsigned char *)decrypted_2.get());
+ ASSERT_TRUE(ret_code > 0);
+ std::string decrypted_content_2(decrypted_2.get(), ret_code);
+ ASSERT_EQ(source_2, decrypted_content_2);
+}
+
+}
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/run-ut.sh b/run-ut.sh
index f2c4779a..9736799f 100755
--- a/run-ut.sh
+++ b/run-ut.sh
@@ -148,6 +148,7 @@ ${DORIS_TEST_BINARY_DIR}/util/types_test
${DORIS_TEST_BINARY_DIR}/util/json_util_test
${DORIS_TEST_BINARY_DIR}/util/byte_buffer_test2
${DORIS_TEST_BINARY_DIR}/util/uid_util_test
+${DORIS_TEST_BINARY_DIR}/util/aes_util_test
## Running common Unittest
${DORIS_TEST_BINARY_DIR}/common/resource_tls_test
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@doris.apache.org
For additional commands, e-mail: dev-help@doris.apache.org