You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2021/05/19 01:36:54 UTC
[impala] 04/05: IMPALA-10679: Add builtin functions to comptute
SHA-1 and SHA-2 digest.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 86beb2f9dd1454e25916b723dce3af23de2ac9e0
Author: Amogh Margoor <am...@cloudera.com>
AuthorDate: Thu May 13 21:04:25 2021 +0100
IMPALA-10679: Add builtin functions to comptute SHA-1 and SHA-2 digest.
Built-in functions to compute SHA-1 digest and SHA-2 family of digest
has been added. Support for SHA2 digest includes SHA224, SHA256,
SHA384 and SHA512. In FIPS mode SHA1, SHA224 and SHA256 have been
disabled and will throw error. SHA2 functions will also throw error
for unsupported bit length i.e., bit length apart from 224, 256, 384,
512.
Testing:
1. Added Unit test for expressions.
2. Added end-to-end test for new functions.
Change-Id: If163b7abda17cca3074c86519d59bcfc6ace21be
Reviewed-on: http://gerrit.cloudera.org:8080/17464
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exprs/expr-test.cc | 83 ++++++++++++++++++++++
be/src/exprs/utility-functions-ir.cc | 71 +++++++++++++++++-
be/src/exprs/utility-functions.h | 9 +++
common/function-registry/impala_functions.py | 4 ++
.../functional-query/queries/QueryTest/exprs.test | 15 ++++
5 files changed, 181 insertions(+), 1 deletion(-)
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 9e801a4..8e3221a 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -30,6 +30,7 @@
#include <boost/unordered_map.hpp>
#include <openssl/crypto.h>
+#include <openssl/sha.h>
#include "codegen/llvm-codegen.h"
#include "common/init.h"
@@ -5529,6 +5530,88 @@ TEST_P(ExprTest, MurmurHashFunction) {
TestIsNull("murmur_hash(NULL)", TYPE_BIGINT);
}
+/// Convert character array `str` of length `len` to hexadecimal.
+std::string ToHex(const unsigned char * str, int len) {
+ stringstream ss;
+ ss << hex << std::uppercase << setfill('0');
+ for (int i = 0; i < len; ++i) {
+ // setw is not sticky. stringstream only converts integral values,
+ // so a cast to int is required, but only convert the least significant byte to hex.
+ ss << setw(2) << (static_cast<int32_t>(str[i]) & 0xFF);
+ }
+ std::string result = ss.str();
+ boost::to_lower(result);
+ return result;
+}
+
+TEST_P(ExprTest, SHAFunctions) {
+ unsigned char input[] = "compute sha digest";
+ std::string sha1fn = std::string("sha1('compute sha digest')");
+ std::string sha2fn = std::string("sha2('compute sha digest'");
+ std::string expected;
+ unsigned char sha1[SHA_DIGEST_LENGTH];
+ unsigned char sha224[SHA224_DIGEST_LENGTH];
+ unsigned char sha256[SHA256_DIGEST_LENGTH];
+ unsigned char sha384[SHA384_DIGEST_LENGTH];
+ unsigned char sha512[SHA512_DIGEST_LENGTH];
+
+ if (FIPS_mode()) {
+ TestError(sha1fn);
+ TestError(sha2fn + ", 224)");
+ TestError(sha2fn + ", 256)");
+ } else {
+ SHA1(input, 18, sha1);
+ expected = ToHex(sha1, SHA_DIGEST_LENGTH);
+ TestStringValue(sha1fn, expected);
+
+ SHA224(input, 18, sha224);
+ expected = ToHex(sha224, SHA224_DIGEST_LENGTH);
+ TestStringValue(sha2fn + ", 224)", expected);
+
+ SHA256(input, 18, sha256);
+ expected = ToHex(sha256, SHA256_DIGEST_LENGTH);
+ TestStringValue(sha2fn + ", 256)", expected);
+ }
+
+ SHA384(input, 18, sha384);
+ expected = ToHex(sha384, SHA384_DIGEST_LENGTH);
+ TestStringValue(sha2fn + ", 384)", expected);
+
+ SHA512(input, 18, sha512);
+ expected = ToHex(sha512, SHA512_DIGEST_LENGTH);
+ TestStringValue(sha2fn + ", 512)", expected);
+
+ // Test empty strings. Empty string is valid input and produces hash.
+ if (!FIPS_mode()) {
+ SHA1(input, 0, sha1);
+ expected = ToHex(sha1, SHA_DIGEST_LENGTH);
+ TestStringValue("sha1('')", expected);
+
+ SHA224(input, 0, sha224);
+ expected = ToHex(sha224, SHA224_DIGEST_LENGTH);
+ TestStringValue("sha2('', 224)", expected);
+
+ SHA256(input, 0, sha256);
+ expected = ToHex(sha256, SHA256_DIGEST_LENGTH);
+ TestStringValue("sha2('', 256)", expected);
+ }
+
+ SHA384(input, 0, sha384);
+ expected = ToHex(sha384, SHA384_DIGEST_LENGTH);
+ TestStringValue("sha2('', 384)", expected);
+
+ SHA512(input, 0, sha512);
+ expected = ToHex(sha512, SHA512_DIGEST_LENGTH);
+ TestStringValue("sha2('', 512)", expected);
+
+ // Test Invalid Inputs
+ TestIsNull("sha1(NULL)", TYPE_STRING);
+ TestIsNull("sha2(NULL, 512)", TYPE_STRING);
+ TestIsNull("sha2('foo', NULL)", TYPE_STRING);
+ // 300 is invalid bit length
+ TestError(sha2fn + ", 300)");
+}
+
TEST_P(ExprTest, SessionFunctions) {
enum Session {S1, S2};
enum Query {Q1, Q2};
diff --git a/be/src/exprs/utility-functions-ir.cc b/be/src/exprs/utility-functions-ir.cc
index bfc143d..1dfa80a 100644
--- a/be/src/exprs/utility-functions-ir.cc
+++ b/be/src/exprs/utility-functions-ir.cc
@@ -17,8 +17,14 @@
#include "exprs/utility-functions.h"
#include <gutil/strings/substitute.h>
-
+#include "common/compiler-util.h"
#include "exprs/anyval-util.h"
+#include "exprs/math-functions.h"
+#include "exprs/string-functions.h"
+
+#include <openssl/crypto.h>
+#include <openssl/evp.h>
+#include <openssl/sha.h>
#include "runtime/runtime-state.h"
#include "udf/udf-internal.h"
#include "util/debug-util.h"
@@ -227,4 +233,67 @@ template StringVal UtilityFunctions::TypeOf(
FunctionContext* ctx, const DecimalVal& input_val);
template StringVal UtilityFunctions::TypeOf(
FunctionContext* ctx, const DateVal& input_val);
+
+StringVal UtilityFunctions::Sha1(FunctionContext* ctx, const StringVal& input_str) {
+ // Validate the input string.
+ if (input_str.is_null) {
+ return StringVal::null();
+ }
+
+ // SHA-1 is not supported for FIPS.
+ if (FIPS_mode()) {
+ ctx->SetError("sha1 is not supported in FIPS mode.");
+ return StringVal::null();
+ } else {
+ StringVal sha1_hash(ctx, SHA_DIGEST_LENGTH);
+ if (UNLIKELY(sha1_hash.is_null)) return StringVal::null();
+ SHA1(input_str.ptr, input_str.len, sha1_hash.ptr);
+ return StringFunctions::Lower(ctx, MathFunctions::HexString(ctx, sha1_hash));
+ }
+}
+
+StringVal UtilityFunctions::Sha2(FunctionContext* ctx, const StringVal& input_str,
+ const IntVal& bit_len) {
+ // Validate the input string.
+ if (input_str.is_null || bit_len.is_null) {
+ return StringVal::null();
+ }
+
+ // SHA-224 and SHA-256 are deprecated for FIPS mode.
+ if (FIPS_mode() && (bit_len.val == 224 || bit_len.val == 256)) {
+ ctx->SetError("Only bit lengths 384 and 512 are supported in FIPS mode.");
+ return StringVal::null();
+ }
+
+ StringVal sha_hash;
+
+ switch(bit_len.val) {
+ case 224:
+ sha_hash = StringVal(ctx, SHA224_DIGEST_LENGTH);
+ if (UNLIKELY(sha_hash.is_null)) return StringVal::null();
+ SHA224(input_str.ptr, input_str.len, sha_hash.ptr);
+ break;
+ case 256:
+ sha_hash = StringVal(ctx, SHA256_DIGEST_LENGTH);
+ if (UNLIKELY(sha_hash.is_null)) return StringVal::null();
+ SHA256(input_str.ptr, input_str.len, sha_hash.ptr);
+ break;
+ case 384:
+ sha_hash = StringVal(ctx, SHA384_DIGEST_LENGTH);
+ if (UNLIKELY(sha_hash.is_null)) return StringVal::null();
+ SHA384(input_str.ptr, input_str.len, sha_hash.ptr);
+ break;
+ case 512:
+ sha_hash = StringVal(ctx, SHA512_DIGEST_LENGTH);
+ if (UNLIKELY(sha_hash.is_null)) return StringVal::null();
+ SHA512(input_str.ptr, input_str.len, sha_hash.ptr);
+ break;
+ default:
+ // Unsupported bit length.
+ ctx->SetError(Substitute("Bit Length $0 is not supported", bit_len.val).c_str());
+ return StringVal::null();
+ }
+
+ return StringFunctions::Lower(ctx, MathFunctions::HexString(ctx, sha_hash));
+}
}
diff --git a/be/src/exprs/utility-functions.h b/be/src/exprs/utility-functions.h
index b50f7f3..2b26541 100644
--- a/be/src/exprs/utility-functions.h
+++ b/be/src/exprs/utility-functions.h
@@ -101,6 +101,15 @@ class UtilityFunctions {
/// the compiler generate the corresponding fully-qualified function name.
template <typename T> static StringVal TypeOf(FunctionContext* ctx, const T& input_val);
+ /// Implementation of sha1 function. Returns the SHA-1 digest for a given string.
+ static StringVal Sha1(FunctionContext* ctx, const StringVal& input_str);
+
+ /// Implementation of sha2 function. Returns the SHA-2 family of hash
+ /// functions (SHA-224, SHA-256, SHA-384, and SHA-512) for a given string.
+ /// 'bit_len' specifies the length of bit. Allowed values are: 224, 256, 384, 512.
+ static StringVal Sha2(FunctionContext* ctx, const StringVal& input_str,
+ const IntVal& bit_len);
+
private:
static StringVal GenUuid(FunctionContext* ctx);
};
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index ffd2f61..4871914 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -738,6 +738,10 @@ visible_functions = [
'_ZN6impala16UtilityFunctions19MurmurHashTimestampEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['murmur_hash'], 'BIGINT', ['DECIMAL'],
'_ZN6impala16UtilityFunctions17MurmurHashDecimalEPN10impala_udf15FunctionContextERKNS1_10DecimalValE'],
+ [['sha1'], 'STRING', ['STRING'],
+ '_ZN6impala16UtilityFunctions4Sha1EPN10impala_udf15FunctionContextERKNS1_9StringValE'],
+ [['sha2'], 'STRING', ['STRING', 'INT'],
+ '_ZN6impala16UtilityFunctions4Sha2EPN10impala_udf15FunctionContextERKNS1_9StringValERKNS1_6IntValE'],
# (Non)NullValue functions
[['nullvalue'], 'BOOLEAN', ['BOOLEAN'], '_ZN6impala15IsNullPredicate6IsNullIN10impala_udf10BooleanValEEES3_PNS2_15FunctionContextERKT_'],
diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
index 7d7e4fe..d57037d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
@@ -3119,3 +3119,18 @@ true,'TEST0'
---- TYPES
BOOLEAN, STRING
====
+---- QUERY
+select sha1(s) from functional.strings_with_quotes where i = 10;
+---- RESULTS
+'dd34bd3ce9aa419f49921e5ff2e67210fe04a4e2'
+---- TYPES
+STRING
+====
+---- QUERY
+select sha2(s, 224), sha2(s, 256), sha2(s, 384),
+sha2(s, 512) from functional.strings_with_quotes where i = 10;
+---- RESULTS
+'c77557470807461a5f30c8eb5fc93147c78fa41261714d495eb871ec','738f748710e0adc2e62f2e5f9b5cec4ed12abfd7142c0bd4ff81966e5a17a278','afef2f6068e8a80c58ce141bf661255e696e5d6c436485212b0190fc36b5465ec4e86d6492e9fe94f6392462fb5fc06c','225b4d74a7697d2064ac6ff8d71bdfddea62d75ea495274fbc55dc39ff0800c85bce11992398d5dd7eabd2deffdbb56822fbb995666b78e449b2ab88ff7c8d86'
+---- TYPES
+STRING, STRING, STRING, STRING
+====
\ No newline at end of file