You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2020/03/08 19:04:48 UTC
[bigtop] branch branch-1.4 updated (d6fec66 -> 4c7d1f8)
This is an automated email from the ASF dual-hosted git repository.
oflebbe pushed a change to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/bigtop.git.
from d6fec66 Bigtop 1.4.0 website update
new 7a5e779 BIGTOP-3308: add support for openssl 1.1.0 on Debian
new 4c7d1f8 BIGTOP-3323: Fix hadoop pom for 1.4 branch
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/common/hadoop/patch3-HADOOP-15062.diff | 105 +++++++++++++++++++++
.../src/common/hadoop/patch4-BIGTOP3323-pom.diff | 11 +++
2 files changed, 116 insertions(+)
create mode 100644 bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
create mode 100644 bigtop-packages/src/common/hadoop/patch4-BIGTOP3323-pom.diff
[bigtop] 02/02: BIGTOP-3323: Fix hadoop pom for 1.4 branch
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oflebbe pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/bigtop.git
commit 4c7d1f84ef2cea9fea79a6420ccbe88a35abd6e8
Author: Olaf Flebbe <of...@oflebbe.de>
AuthorDate: Sun Mar 8 20:04:08 2020 +0100
BIGTOP-3323: Fix hadoop pom for 1.4 branch
---
bigtop-packages/src/common/hadoop/patch4-BIGTOP3323-pom.diff | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/bigtop-packages/src/common/hadoop/patch4-BIGTOP3323-pom.diff b/bigtop-packages/src/common/hadoop/patch4-BIGTOP3323-pom.diff
new file mode 100644
index 0000000..f6f60a4
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch4-BIGTOP3323-pom.diff
@@ -0,0 +1,11 @@
+--- hadoop-2.8.5/pom.xml_old 2020-02-23 16:12:14.956399393 +0000
++++ hadoop-2.8.5/pom.xml 2020-02-23 16:12:38.116835137 +0000
+@@ -59,7 +59,7 @@
+ </repository>
+ <repository>
+ <id>repository.jboss.org</id>
+- <url>http://repository.jboss.org/nexus/content/groups/public/</url>
++ <url>https://repository.jboss.org/nexus/content/groups/public/</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
[bigtop] 01/02: BIGTOP-3308: add support for openssl 1.1.0 on Debian
Posted by of...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oflebbe pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/bigtop.git
commit 7a5e779ba7437f7b427a50ba724120a455447bb3
Author: Luca Toscano <lt...@wikimedia.org>
AuthorDate: Sat Feb 29 11:03:23 2020 +0100
BIGTOP-3308: add support for openssl 1.1.0 on Debian
In BIGTOP-2932 a patch was backported from Hadoop upstream to better
support openssl 1.1.x. Some time later a new commit was filed to address
some corner cases, but it was never backported to BigTop.
On Debian 9 openssl 1.1.0 is the default, and BigTop builds against it.
When executing the command "hadoop checknative -a" though an error shows up:
openssl: false EVP_CIPHER_CTX_cleanup
The issue is that EVP_CIPHER_CTX_cleanup was removed from openssl 1.1.0, but
OpensslCipher.c kept using it until a proper if guard was added by the
patch provided by this pull request.
In my case this issue caused Spark 2.4's RPC encryption, based on Openssl native
lib, to fail when establishing new connections between workers and AM on Yarn.
I rebuilt the hadoop deb package with the patch and tested in the Debian 9 Docker
image, the checknative util now returns correctly a positive "true" for openssl.
I also tested it with Spark 2.4 on a test Hadoop cluster and it seems working fine.
---
.../src/common/hadoop/patch3-HADOOP-15062.diff | 105 +++++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git a/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
new file mode 100644
index 0000000..e6b5456
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
@@ -0,0 +1,105 @@
+From 138c1ed5660f713d24bfebc44ea1846f76c00cb9 Mon Sep 17 00:00:00 2001
+From: Yufei Gu <yu...@apache.org>
+Date: Tue, 20 Mar 2018 15:19:18 -0700
+Subject: [PATCH] HADOOP-15062. TestCryptoStreamsWithOpensslAesCtrCryptoCodec
+ fails on Debian 9. Contributed by Miklos Szegedi.
+
+---
+ .../org/apache/hadoop/crypto/OpensslCipher.c | 33 +++++++++++++++++++
+ 1 file changed, 33 insertions(+)
+
+diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+index c7984a33475f..abff7ea5f17f 100644
+--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+@@ -27,8 +27,12 @@
+ #ifdef UNIX
+ static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
+ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++static int (*dlsym_EVP_CIPHER_CTX_reset)(EVP_CIPHER_CTX *);
++#else
+ static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
+ static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
++#endif
+ static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
+ static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
+ static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
+@@ -123,10 +127,16 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+ "EVP_CIPHER_CTX_new");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \
+ "EVP_CIPHER_CTX_free");
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_reset, env, openssl, \
++ "EVP_CIPHER_CTX_reset");
++#else
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \
+ "EVP_CIPHER_CTX_cleanup");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \
+ "EVP_CIPHER_CTX_init");
++#endif
++
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \
+ "EVP_CIPHER_CTX_set_padding");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \
+@@ -271,7 +281,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
+ (*env)->ReleaseByteArrayElements(env, key, jKey, 0);
+ (*env)->ReleaseByteArrayElements(env, iv, jIv, 0);
+ if (rc == 0) {
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ dlsym_EVP_CIPHER_CTX_reset(context);
++#else
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
++#endif
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex.");
+ return (jlong)0;
+ }
+@@ -334,7 +348,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
+ int output_len = 0;
+ if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \
+ input_bytes, input_len)) {
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ dlsym_EVP_CIPHER_CTX_reset(context);
++#else
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
++#endif
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate.");
+ return 0;
+ }
+@@ -376,7 +394,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal
+
+ int output_len = 0;
+ if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) {
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ dlsym_EVP_CIPHER_CTX_reset(context);
++#else
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
++#endif
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex.");
+ return 0;
+ }
+@@ -396,6 +418,16 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary
+ (JNIEnv *env, jclass clazz)
+ {
+ #ifdef UNIX
++#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
++ if (dlsym_EVP_CIPHER_CTX_reset) {
++ Dl_info dl_info;
++ if(dladdr(
++ dlsym_EVP_CIPHER_CTX_reset,
++ &dl_info)) {
++ return (*env)->NewStringUTF(env, dl_info.dli_fname);
++ }
++ }
++#else
+ if (dlsym_EVP_CIPHER_CTX_init) {
+ Dl_info dl_info;
+ if(dladdr(
+@@ -404,6 +436,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary
+ return (*env)->NewStringUTF(env, dl_info.dli_fname);
+ }
+ }
++#endif
+
+ return (*env)->NewStringUTF(env, HADOOP_OPENSSL_LIBRARY);
+ #endif