You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by se...@apache.org on 2020/11/11 23:10:39 UTC
[bigtop] branch master updated: BIGTOP-3444. Bump Hadoop to 2.10.1.
(#693)
This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new d661a56 BIGTOP-3444. Bump Hadoop to 2.10.1. (#693)
d661a56 is described below
commit d661a5643471c9fd1057e7979670a08256659a41
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Thu Nov 12 07:54:35 2020 +0900
BIGTOP-3444. Bump Hadoop to 2.10.1. (#693)
* BIGTOP-3444. Bump Hadoop to 2.10.1.
* addressed API change of azure-storage.
---
.../src/common/hadoop/patch1-YARN-9945.diff | 18 ---
.../src/common/hadoop/patch2-HADOOP-14597.diff | 143 ---------------------
.../src/common/hadoop/patch3-HADOOP-15062.diff | 105 ---------------
.../src/common/hadoop/patch3-HADOOP-16739.diff | 40 ------
.../src/common/hadoop/patch4-HADOOP-16647.diff | 81 ------------
.../zeppelin/patch3-azure-storage-7.0.1.diff | 13 ++
bigtop.bom | 2 +-
7 files changed, 14 insertions(+), 388 deletions(-)
diff --git a/bigtop-packages/src/common/hadoop/patch1-YARN-9945.diff b/bigtop-packages/src/common/hadoop/patch1-YARN-9945.diff
deleted file mode 100644
index fb8eba3..0000000
--- a/bigtop-packages/src/common/hadoop/patch1-YARN-9945.diff
+++ /dev/null
@@ -1,18 +0,0 @@
-commit ae3f5d66e4fac859f31a66870e079c91ba8993d9
-Author: Jonathan Hung <jh...@linkedin.com>
-Date: Thu Oct 31 15:30:23 2019 -0700
-
- YARN-9945. Fix javadoc in FederationProxyProviderUtil in branch-2
-
-diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/Feder
-index 33883b9..91924da 100644
---- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
-+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
-@@ -28,6 +28,7 @@
- import org.apache.hadoop.security.token.Token;
- import org.apache.hadoop.security.token.TokenIdentifier;
- import org.apache.hadoop.yarn.client.AMRMClientUtils;
-+import org.apache.hadoop.yarn.client.ClientRMProxy;
- import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
- import org.apache.hadoop.yarn.conf.HAUtil;
- import org.apache.hadoop.yarn.conf.YarnConfiguration;
diff --git a/bigtop-packages/src/common/hadoop/patch2-HADOOP-14597.diff b/bigtop-packages/src/common/hadoop/patch2-HADOOP-14597.diff
deleted file mode 100644
index cbd428d..0000000
--- a/bigtop-packages/src/common/hadoop/patch2-HADOOP-14597.diff
+++ /dev/null
@@ -1,143 +0,0 @@
-commit 94ca52ae9ec0ae04854d726bf2ac1bc457b96a9c
-Author: Ravi Prakash <ra...@apache.org>
-Date: Mon Jul 24 16:01:45 2017 -0700
-
- HADOOP-14597. Native compilation broken with OpenSSL-1.1.0. Contributed by Ravi Prakash.
-
-diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
-index 5cb5bba..c7984a3 100644
---- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
-+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
-@@ -30,6 +30,11 @@ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
- static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
- static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
- static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
-+static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
-+static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
-+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-+static int (*dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *);
-+#endif
- static int (*dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, const EVP_CIPHER *, \
- ENGINE *, const unsigned char *, const unsigned char *, int);
- static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *, \
-@@ -46,6 +51,11 @@ typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
- typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
- typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
- typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
-+typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
-+typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
-+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-+typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *);
-+#endif
- typedef int (__cdecl *__dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, \
- const EVP_CIPHER *, ENGINE *, const unsigned char *, \
- const unsigned char *, int);
-@@ -60,6 +70,11 @@ static __dlsym_EVP_CIPHER_CTX_free dlsym_EVP_CIPHER_CTX_free;
- static __dlsym_EVP_CIPHER_CTX_cleanup dlsym_EVP_CIPHER_CTX_cleanup;
- static __dlsym_EVP_CIPHER_CTX_init dlsym_EVP_CIPHER_CTX_init;
- static __dlsym_EVP_CIPHER_CTX_set_padding dlsym_EVP_CIPHER_CTX_set_padding;
-+static __dlsym_EVP_CIPHER_CTX_test_flags dlsym_EVP_CIPHER_CTX_test_flags;
-+static __dlsym_EVP_CIPHER_CTX_block_size dlsym_EVP_CIPHER_CTX_block_size;
-+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-+static __dlsym_EVP_CIPHER_CTX_encrypting dlsym_EVP_CIPHER_CTX_encrypting;
-+#endif
- static __dlsym_EVP_CipherInit_ex dlsym_EVP_CipherInit_ex;
- static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate;
- static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex;
-@@ -114,6 +129,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
- "EVP_CIPHER_CTX_init");
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \
- "EVP_CIPHER_CTX_set_padding");
-+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \
-+ "EVP_CIPHER_CTX_test_flags");
-+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \
-+ "EVP_CIPHER_CTX_block_size");
-+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \
-+ "EVP_CIPHER_CTX_encrypting");
-+#endif
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \
- "EVP_CipherInit_ex");
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherUpdate, env, openssl, \
-@@ -135,6 +158,17 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
- LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_set_padding, \
- dlsym_EVP_CIPHER_CTX_set_padding, env, \
- openssl, "EVP_CIPHER_CTX_set_padding");
-+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_test_flags, \
-+ dlsym_EVP_CIPHER_CTX_test_flags, env, \
-+ openssl, "EVP_CIPHER_CTX_test_flags");
-+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \
-+ dlsym_EVP_CIPHER_CTX_block_size, env, \
-+ openssl, "EVP_CIPHER_CTX_block_size");
-+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \
-+ dlsym_EVP_CIPHER_CTX_encrypting, env, \
-+ openssl, "EVP_CIPHER_CTX_encrypting");
-+#endif
- LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \
- env, openssl, "EVP_CipherInit_ex");
- LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherUpdate, dlsym_EVP_CipherUpdate, \
-@@ -253,14 +287,18 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
- static int check_update_max_output_len(EVP_CIPHER_CTX *context, int input_len,
- int max_output_len)
- {
-- if (context->flags & EVP_CIPH_NO_PADDING) {
-+ if ( dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) {
- if (max_output_len >= input_len) {
- return 1;
- }
- return 0;
- } else {
-- int b = context->cipher->block_size;
-+ int b = dlsym_EVP_CIPHER_CTX_block_size(context);
-+#if OPENSSL_VERSION_NUMBER < 0x10100000L
- if (context->encrypt) {
-+#else
-+ if (dlsym_EVP_CIPHER_CTX_encrypting(context)) {
-+#endif
- if (max_output_len >= input_len + b - 1) {
- return 1;
- }
-@@ -307,10 +345,10 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
- static int check_doFinal_max_output_len(EVP_CIPHER_CTX *context,
- int max_output_len)
- {
-- if (context->flags & EVP_CIPH_NO_PADDING) {
-+ if ( dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) {
- return 1;
- } else {
-- int b = context->cipher->block_size;
-+ int b = dlsym_EVP_CIPHER_CTX_block_size(context);
- if (max_output_len >= b) {
- return 1;
- }
-diff --git a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc
-index 91fb5a4..45cb8c2 100644
---- a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc
-+++ b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc
-@@ -420,6 +420,7 @@ namespace HadoopPipes {
- }
-
- string createDigest(string &password, string& msg) {
-+#if OPENSSL_VERSION_NUMBER < 0x10100000L
- HMAC_CTX ctx;
- unsigned char digest[EVP_MAX_MD_SIZE];
- HMAC_Init(&ctx, (const unsigned char *)password.c_str(),
-@@ -428,7 +429,16 @@ namespace HadoopPipes {
- unsigned int digestLen;
- HMAC_Final(&ctx, digest, &digestLen);
- HMAC_cleanup(&ctx);
--
-+#else
-+ HMAC_CTX *ctx = HMAC_CTX_new();
-+ unsigned char digest[EVP_MAX_MD_SIZE];
-+ HMAC_Init_ex(ctx, (const unsigned char *)password.c_str(),
-+ password.length(), EVP_sha1(), NULL);
-+ HMAC_Update(ctx, (const unsigned char *)msg.c_str(), msg.length());
-+ unsigned int digestLen;
-+ HMAC_Final(ctx, digest, &digestLen);
-+ HMAC_CTX_free(ctx);
-+#endif
- //now apply base64 encoding
- BIO *bmem, *b64;
- BUF_MEM *bptr;
diff --git a/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff b/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
deleted file mode 100644
index e6b5456..0000000
--- a/bigtop-packages/src/common/hadoop/patch3-HADOOP-15062.diff
+++ /dev/null
@@ -1,105 +0,0 @@
-From 138c1ed5660f713d24bfebc44ea1846f76c00cb9 Mon Sep 17 00:00:00 2001
-From: Yufei Gu <yu...@apache.org>
-Date: Tue, 20 Mar 2018 15:19:18 -0700
-Subject: [PATCH] HADOOP-15062. TestCryptoStreamsWithOpensslAesCtrCryptoCodec
- fails on Debian 9. Contributed by Miklos Szegedi.
-
----
- .../org/apache/hadoop/crypto/OpensslCipher.c | 33 +++++++++++++++++++
- 1 file changed, 33 insertions(+)
-
-diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
-index c7984a33475f..abff7ea5f17f 100644
---- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
-+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
-@@ -27,8 +27,12 @@
- #ifdef UNIX
- static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
- static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
-+#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
-+static int (*dlsym_EVP_CIPHER_CTX_reset)(EVP_CIPHER_CTX *);
-+#else
- static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
- static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
-+#endif
- static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
- static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
- static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
-@@ -123,10 +127,16 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
- "EVP_CIPHER_CTX_new");
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \
- "EVP_CIPHER_CTX_free");
-+#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_reset, env, openssl, \
-+ "EVP_CIPHER_CTX_reset");
-+#else
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \
- "EVP_CIPHER_CTX_cleanup");
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \
- "EVP_CIPHER_CTX_init");
-+#endif
-+
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \
- "EVP_CIPHER_CTX_set_padding");
- LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \
-@@ -271,7 +281,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
- (*env)->ReleaseByteArrayElements(env, key, jKey, 0);
- (*env)->ReleaseByteArrayElements(env, iv, jIv, 0);
- if (rc == 0) {
-+#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ dlsym_EVP_CIPHER_CTX_reset(context);
-+#else
- dlsym_EVP_CIPHER_CTX_cleanup(context);
-+#endif
- THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex.");
- return (jlong)0;
- }
-@@ -334,7 +348,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
- int output_len = 0;
- if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \
- input_bytes, input_len)) {
-+#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ dlsym_EVP_CIPHER_CTX_reset(context);
-+#else
- dlsym_EVP_CIPHER_CTX_cleanup(context);
-+#endif
- THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate.");
- return 0;
- }
-@@ -376,7 +394,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal
-
- int output_len = 0;
- if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) {
-+#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ dlsym_EVP_CIPHER_CTX_reset(context);
-+#else
- dlsym_EVP_CIPHER_CTX_cleanup(context);
-+#endif
- THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex.");
- return 0;
- }
-@@ -396,6 +418,16 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary
- (JNIEnv *env, jclass clazz)
- {
- #ifdef UNIX
-+#if OPENSSL_API_COMPAT < 0x10100000L && OPENSSL_VERSION_NUMBER >= 0x10100000L
-+ if (dlsym_EVP_CIPHER_CTX_reset) {
-+ Dl_info dl_info;
-+ if(dladdr(
-+ dlsym_EVP_CIPHER_CTX_reset,
-+ &dl_info)) {
-+ return (*env)->NewStringUTF(env, dl_info.dli_fname);
-+ }
-+ }
-+#else
- if (dlsym_EVP_CIPHER_CTX_init) {
- Dl_info dl_info;
- if(dladdr(
-@@ -404,6 +436,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary
- return (*env)->NewStringUTF(env, dl_info.dli_fname);
- }
- }
-+#endif
-
- return (*env)->NewStringUTF(env, HADOOP_OPENSSL_LIBRARY);
- #endif
diff --git a/bigtop-packages/src/common/hadoop/patch3-HADOOP-16739.diff b/bigtop-packages/src/common/hadoop/patch3-HADOOP-16739.diff
deleted file mode 100644
index 790ce26..0000000
--- a/bigtop-packages/src/common/hadoop/patch3-HADOOP-16739.diff
+++ /dev/null
@@ -1,40 +0,0 @@
-diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
-index 3b0b28c5ef1..7e0f48eafb4 100644
---- a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
-+++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
-@@ -22,6 +22,25 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project
- include(HadoopCommon)
-
- find_package(OpenSSL REQUIRED)
-+find_package(PkgConfig QUIET)
-+pkg_check_modules(LIBTIRPC libtirpc)
-+
-+find_path(RPC_INCLUDE_DIRS NAMES rpc/rpc.h)
-+
-+if (NOT RPC_INCLUDE_DIRS)
-+ find_path(TIRPC_INCLUDE_DIRS
-+ NAMES netconfig.h
-+ PATH_SUFFIXES tirpc
-+ HINTS ${LIBTIRPC_INCLUDE_DIRS}
-+ )
-+
-+ find_library(TIRPC_LIBRARIES
-+ NAMES tirpc
-+ HINTS ${LIBTIRPC_LIBRARY_DIRS}
-+ )
-+
-+ include_directories(${TIRPC_INCLUDE_DIRS})
-+endif()
-
- include_directories(
- main/native/utils/api
-@@ -51,6 +70,9 @@ add_library(hadooputils STATIC
- main/native/utils/impl/StringUtils.cc
- main/native/utils/impl/SerialUtils.cc
- )
-+if (NOT RPC_INCLUDE_DIRS AND LIBTIRPC_FOUND)
-+ target_link_libraries(hadooputils tirpc)
-+endif()
-
- add_library(hadooppipes STATIC
- main/native/pipes/impl/HadoopPipes.cc
diff --git a/bigtop-packages/src/common/hadoop/patch4-HADOOP-16647.diff b/bigtop-packages/src/common/hadoop/patch4-HADOOP-16647.diff
deleted file mode 100644
index 358262d..0000000
--- a/bigtop-packages/src/common/hadoop/patch4-HADOOP-16647.diff
+++ /dev/null
@@ -1,81 +0,0 @@
-commit 8f8be6b92a32982c2fd5be4647ebe1610a533dc3
-Author: Masatake Iwasaki <iw...@apache.org>
-Date: Sat Apr 4 07:20:51 2020 +0900
-
- HADOOP-16647. Support OpenSSL 1.1.1 LTS. Contributed by Rakesh Radhakrishnan.
-
-diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
-index 26e1fa6..3f141be 100644
---- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
-+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
-@@ -42,16 +42,18 @@
- #ifdef UNIX
- static void * (*dlsym_CRYPTO_malloc) (int, const char *, int);
- static void (*dlsym_CRYPTO_free) (void *);
-+#if OPENSSL_VERSION_NUMBER < 0x10100000L
- static int (*dlsym_CRYPTO_num_locks) (void);
- static void (*dlsym_CRYPTO_set_locking_callback) (void (*)());
- static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)());
- static void (*dlsym_ENGINE_load_rdrand) (void);
-+static void (*dlsym_ENGINE_cleanup) (void);
-+#endif
- static ENGINE * (*dlsym_ENGINE_by_id) (const char *);
- static int (*dlsym_ENGINE_init) (ENGINE *);
- static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int);
- static int (*dlsym_ENGINE_finish) (ENGINE *);
- static int (*dlsym_ENGINE_free) (ENGINE *);
--static void (*dlsym_ENGINE_cleanup) (void);
- static int (*dlsym_RAND_bytes) (unsigned char *, int);
- static unsigned long (*dlsym_ERR_get_error) (void);
- #endif
-@@ -113,6 +115,8 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_
- dlerror(); // Clear any existing error
- LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_malloc, env, openssl, "CRYPTO_malloc");
- LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_free, env, openssl, "CRYPTO_free");
-+#if OPENSSL_VERSION_NUMBER < 0x10100000L
-+ // pre-1.1.0
- LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_num_locks, env, openssl, "CRYPTO_num_locks");
- LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_set_locking_callback, \
- env, openssl, "CRYPTO_set_locking_callback");
-@@ -120,13 +124,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_
- openssl, "CRYPTO_set_id_callback");
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_load_rdrand, env, \
- openssl, "ENGINE_load_rdrand");
-+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup");
-+#endif
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id");
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init");
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \
- openssl, "ENGINE_set_default");
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish");
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free");
-- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup");
- LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes");
- LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error");
- #endif
-@@ -303,9 +308,11 @@ static unsigned long pthreads_thread_id(void)
- */
- static ENGINE * openssl_rand_init(void)
- {
-+#if OPENSSL_VERSION_NUMBER < 0x10100000L
- locks_setup();
-
- dlsym_ENGINE_load_rdrand();
-+#endif
- ENGINE *eng = dlsym_ENGINE_by_id("rdrand");
-
- int ret = -1;
-@@ -340,11 +347,12 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks)
- dlsym_ENGINE_finish(eng);
- dlsym_ENGINE_free(eng);
- }
--
-+#if OPENSSL_VERSION_NUMBER < 0x10100000L
- dlsym_ENGINE_cleanup();
- if (clean_locks) {
- locks_cleanup();
- }
-+#endif
- }
-
- static int openssl_rand_bytes(unsigned char *buf, int num)
diff --git a/bigtop-packages/src/common/zeppelin/patch3-azure-storage-7.0.1.diff b/bigtop-packages/src/common/zeppelin/patch3-azure-storage-7.0.1.diff
new file mode 100644
index 0000000..518c975
--- /dev/null
+++ b/bigtop-packages/src/common/zeppelin/patch3-azure-storage-7.0.1.diff
@@ -0,0 +1,13 @@
+diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
+index 3b0108890..679f2513a 100644
+--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
++++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
+@@ -160,7 +160,7 @@ public class AzureNotebookRepo implements NotebookRepo {
+ }
+
+ // unfortunately, we need to use a recursive delete here
+- private void delete(ListFileItem item) throws StorageException {
++ private void delete(ListFileItem item) throws StorageException, URISyntaxException {
+ if (item.getClass() == CloudFileDirectory.class) {
+ CloudFileDirectory dir = (CloudFileDirectory) item;
+
diff --git a/bigtop.bom b/bigtop.bom
index 47842ae..8b809ee 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -148,7 +148,7 @@ bigtop {
'hadoop' {
name = 'hadoop'
relNotes = 'Apache Hadoop'
- version { base = '2.10.0'; pkg = base; release = 1 }
+ version { base = '2.10.1'; pkg = base; release = 1 }
tarball { destination = "${name}-${version.base}.tar.gz"
source = "${name}-${version.base}-src.tar.gz" }
url { download_path = "/$name/common/$name-${version.base}"