You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by aw...@apache.org on 2020/12/02 20:32:09 UTC
[kudu] 01/03: [dist_test] Ship security libraries to dist_test
This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch branch-1.12.x
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit e011d26fcb161d449ecb075f997138c314ffe0fe
Author: Grant Henke <gr...@apache.org>
AuthorDate: Thu Nov 12 19:05:11 2020 -0600
[dist_test] Ship security libraries to dist_test
This patch adjusts the lib whitelist to allow shipping the security libraries
to dist_test and allow more flexibility when versions do not match the
dist_test images versions.
This was already happening for rhel6 installs due to the rhel6 workaround script
linked below. With this change the libraries will be shipped even when not
in thirdparty.
https://github.com/apache/kudu/blob/master/thirdparty/install-openssl-el6-workaround.sh
I also needed to adjust run_dist_test.py in order to set the SASL_PATH
environment variable if SASL modules are present. Otherwise the
system modules were still used.
Change-Id: Id10afab6e9c48b9ffcf0da905993c7f2a1e606a6
Reviewed-on: http://gerrit.cloudera.org:8080/16716
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Reviewed-by: Alexey Serbin <as...@cloudera.com>
(cherry picked from commit fab3a38d1da0d5d13d8e9c91fd306e03b65da4e6)
Reviewed-on: http://gerrit.cloudera.org:8080/16801
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Andrew Wong <aw...@cloudera.com>
---
build-support/dist_test.py | 63 +++++++++++++++++++++++++++++++++++++--
build-support/run_dist_test.py | 6 ++++
cmake_modules/FindCyrusSASL.cmake | 2 +-
3 files changed, 68 insertions(+), 3 deletions(-)
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index aa20d70..43cc537 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -241,6 +241,11 @@ def get_test_executions(tests_regex, extra_args=None):
def is_lib_whitelisted(lib):
# No need to ship things like libc, libstdcxx, etc.
if lib.startswith("/lib") or lib.startswith("/usr"):
+ # Ship the dynamically linked security libraries from
+ # OpenSSL and Cyrus SASL to better support submitting
+ # installed versions different from the dist_test image.
+ if "libcrypto" in lib or "libsasl2" in lib or "libssl" in lib:
+ return True
return False
return True
@@ -263,8 +268,41 @@ def get_base_deps(dep_extractor):
# of the test executable. We must include those dependencies in the archive
# for the binaries to be usable.
deps.extend(dep_extractor.extract_deps(d))
+
+ add_sasl_module_deps(deps)
return deps
+def add_sasl_module_deps(deps):
+ """
+ The SASL module dependencies are used at runtime but are not discovered
+ via ldd in the dep_extractor. This method finds the sasl2 directory
+ relative to the libsasl2 library and adds all the libraries in that
+ directory.
+ """
+ # Find the libsasl2 module in the dependencies.
+ sasl_lib = None
+ for dep in deps:
+ if "libsasl2" in dep:
+ sasl_lib = dep
+ break
+
+ # Look for libplain in potential sasl2 module paths, which is required for
+ # Kudu's basic operation.
+ sasl_path = None
+ if sasl_lib:
+ path = os.path.join(os.path.dirname(sasl_lib), "sasl2")
+ if os.path.exists(path):
+ children = os.listdir(path)
+ for child in children:
+ if "libplain" in child:
+ sasl_path = path
+ break
+
+ if sasl_path:
+ for dirpath, subdirs, files in os.walk(sasl_path):
+ for f in files:
+ dep = os.path.join(dirpath, f)
+ deps.append(dep)
def is_outside_of_tree(path):
repo_dir = rel_to_abs("./")
@@ -283,7 +321,18 @@ def copy_system_library(lib):
sys_lib_dir = rel_to_abs("build/dist-test-system-libs")
if not os.path.exists(sys_lib_dir):
os.makedirs(sys_lib_dir)
- dst = os.path.join(sys_lib_dir, os.path.basename(lib))
+
+ sasl_dir = os.path.join(sys_lib_dir, "sasl2")
+ if not os.path.exists(sasl_dir):
+ os.makedirs(sasl_dir)
+
+ # If the library is a SASL module keep it in its own directory so
+ # we can set the SASL_PATH environment variable in run_dist_test.py.
+ if "/sasl2/" in lib:
+ dst = os.path.join(sasl_dir, os.path.basename(lib))
+ else:
+ dst = os.path.join(sys_lib_dir, os.path.basename(lib))
+
# Copy if it doesn't exist, or the mtimes don't match.
# Using shutil.copy2 preserves the mtime after the copy (like cp -p)
if not os.path.exists(dst) or os.stat(dst).st_mtime != os.stat(lib).st_mtime:
@@ -665,7 +714,17 @@ def add_java_subparser(subparsers):
loop.set_defaults(func=loop_java_test)
def dump_base_deps(parser, options):
- print(json.dumps(get_base_deps(create_dependency_extractor())))
+ deps = get_base_deps(create_dependency_extractor())
+ relocated_deps = []
+ # Deduplicate dependencies included via DEPS_FOR_ALL.
+ for d in set(deps):
+ # System libraries will end up being relative paths out
+ # of the build tree. We need to copy those into the build
+ # tree somewhere.
+ if is_outside_of_tree(d):
+ d = copy_system_library(d)
+ relocated_deps.append(d)
+ print(json.dumps(relocated_deps))
def add_internal_commands(subparsers):
p = subparsers.add_parser('internal', help="[Internal commands not for users]")
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index cfb2c75..b8742f5 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -184,6 +184,12 @@ def main():
[os.path.join(ROOT, "build/dist-test-system-libs/")] +
glob.glob(os.path.abspath(os.path.join(ROOT, "build/*/lib"))))
+ # If SASL modules are included in the dist-test-system-libs, set the
+ # SASL_PATH environment variable to use them instead of the system ones.
+ sasl_dir = os.path.join(ROOT, "build/dist-test-system-libs/sasl2")
+ if os.path.exists(sasl_dir):
+ env['SASL_PATH'] = sasl_dir
+
# Don't pollute /tmp in dist-test setting. If a test crashes, the dist-test slave
# will clear up our working directory but won't be able to find and clean up things
# left in /tmp.
diff --git a/cmake_modules/FindCyrusSASL.cmake b/cmake_modules/FindCyrusSASL.cmake
index 8f92cf5..24192b7 100644
--- a/cmake_modules/FindCyrusSASL.cmake
+++ b/cmake_modules/FindCyrusSASL.cmake
@@ -24,7 +24,7 @@
#
# N.B: we do _not_ include sasl in thirdparty, for a fairly subtle reason. The
# TLDR version is that newer versions of cyrus-sasl (>=2.1.26) have a bug fix
-# for https://bugzilla.cyrusimap.org/show_bug.cgi?id=3590, but that bug fix
+# for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=728332, but that bug fix
# relied on a change both on the plugin side and on the library side. If you
# then try to run the new version of sasl (e.g from our thirdparty tree) with
# an older version of a plugin (eg from RHEL6 install), you'll get a SASL_NOMECH