You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2020/03/08 19:04:49 UTC
[bigtop] 01/02: BIGTOP-3308: add support for openssl 1.1.0 on Debian
This is an automated email from the ASF dual-hosted git repository.
oflebbe pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/bigtop.git
commit 7a5e779ba7437f7b427a50ba724120a455447bb3
Author: Luca Toscano <lt...@wikimedia.org>
AuthorDate: Sat Feb 29 11:03:23 2020 +0100
BIGTOP-3308: add support for openssl 1.1.0 on Debian
In BIGTOP-2932 a patch was backported from Hadoop upstream to better
support openssl 1.1.x. Some time later a new commit was filed to address
some corner cases, but it was never backported to BigTop.
On Debian 9 openssl 1.1.0 is the default, and BigTop builds against it.
When executing the command "hadoop checknative -a" though an error shows up:
openssl: false EVP_CIPHER_CTX_cleanup
The issue is that EVP_CIPHER_CTX_cleanup was removed from openssl 1.1.0, but
OpensslCipher.c kept using it until a proper if guard was added by the
patch provided by this pull request.
In my case this issue caused Spark 2.4's RPC encryption, based on Openssl native
lib, to fail when establishing new connections between workers and AM on Yarn.
I rebuilt the hadoop deb package with the patch and tested in the Debian 9 Docker
image, the checknative util now returns correctly a positive "true" for openssl.
I also tested it with Spark 2.4 on a test Hadoop cluster and it seems working fine.
---
.../src/common/hadoop/patch3-HADOOP-15062.diff | 105 +++++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git a/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
new file mode 100644
index 0000000..e6b5456
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
@@ -0,0 +1,105 @@
+From 138c1ed5660f713d24bfebc44ea1846f76c00cb9 Mon Sep 17 00:00:00 2001
+From: Yufei Gu <yu...@apache.org>
+Date: Tue, 20 Mar 2018 15:19:18 -0700
+Subject: [PATCH] HADOOP-15062. TestCryptoStreamsWithOpensslAesCtrCryptoCodec
+ fails on Debian 9. Contributed by Miklos Szegedi.
+
+---
+ .../org/apache/hadoop/crypto/OpensslCipher.c | 33 +++++++++++++++++++
+ 1 file changed, 33 insertions(+)
+
+diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+index c7984a33475f..abff7ea5f17f 100644
+--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+@@ -27,8 +27,12 @@
+ #ifdef UNIX
+ static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
+ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++static int (*dlsym_EVP_CIPHER_CTX_reset)(EVP_CIPHER_CTX *);
++#else
+ static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
+ static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
++#endif
+ static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
+ static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
+ static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
+@@ -123,10 +127,16 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+ "EVP_CIPHER_CTX_new");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \
+ "EVP_CIPHER_CTX_free");
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_reset, env, openssl, \
++ "EVP_CIPHER_CTX_reset");
++#else
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \
+ "EVP_CIPHER_CTX_cleanup");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \
+ "EVP_CIPHER_CTX_init");
++#endif
++
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \
+ "EVP_CIPHER_CTX_set_padding");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \
+@@ -271,7 +281,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
+ (*env)->ReleaseByteArrayElements(env, key, jKey, 0);
+ (*env)->ReleaseByteArrayElements(env, iv, jIv, 0);
+ if (rc == 0) {
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ dlsym_EVP_CIPHER_CTX_reset(context);
++#else
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
++#endif
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex.");
+ return (jlong)0;
+ }
+@@ -334,7 +348,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
+ int output_len = 0;
+ if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \
+ input_bytes, input_len)) {
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ dlsym_EVP_CIPHER_CTX_reset(context);
++#else
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
++#endif
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate.");
+ return 0;
+ }
+@@ -376,7 +394,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal
+
+ int output_len = 0;
+ if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) {
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ dlsym_EVP_CIPHER_CTX_reset(context);
++#else
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
++#endif
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex.");
+ return 0;
+ }
+@@ -396,6 +418,16 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary
+ (JNIEnv *env, jclass clazz)
+ {
+ #ifdef UNIX
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ if (dlsym_EVP_CIPHER_CTX_reset) {
++ Dl_info dl_info;
++ if(dladdr(
++ dlsym_EVP_CIPHER_CTX_reset,
++ &dl_info)) {
++ return (*env)->NewStringUTF(env, dl_info.dli_fname);
++ }
++ }
++#else
+ if (dlsym_EVP_CIPHER_CTX_init) {
+ Dl_info dl_info;
+ if(dladdr(
+@@ -404,6 +436,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary
+ return (*env)->NewStringUTF(env, dl_info.dli_fname);
+ }
+ }
++#endif
+
+ return (*env)->NewStringUTF(env, HADOOP_OPENSSL_LIBRARY);
+ #endif