You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2023/06/15 16:23:14 UTC
[impala] branch master updated (9727b46f3 -> c5ec833ae)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 9727b46f3 IMPALA-11435: Fixup - Suppress logging for 'thrift' in impala-shell
new 9c6df6a69 IMPALA-12179 (part 1): Remove dependency on lsb_release for docker CMake
new f0065cdb7 IMPALA-12179 (part 2): Use /etc/os-release for bootstrap_toolchain.py
new 6222785ef IMPALA-12179 (part 3): Remove remaining lsb_release references
new c5ec833ae IMPALA-12211: Adapt FIPS detection to handle OpenSSL 3
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 2 +-
be/src/exprs/expr-test.cc | 10 ++--
be/src/exprs/mask-functions-ir.cc | 4 +-
be/src/exprs/utility-functions-ir.cc | 9 ++--
be/src/kudu/util/openssl_util.cc | 6 ++-
be/src/kudu/util/openssl_util.h | 4 ++
be/src/util/openssl-util.h | 16 ++++--
be/src/util/webserver-test.cc | 6 +--
be/src/util/webserver.cc | 3 +-
bin/bootstrap_system.sh | 2 +-
bin/bootstrap_toolchain.py | 97 +++++++++++++++++------------------
bin/distcc/distcc_server_bootstrap.sh | 8 +--
bin/distcc/distcc_server_setup.sh | 12 ++---
bin/jenkins/install_docker.sh | 2 +-
docker/CMakeLists.txt | 47 ++++++++---------
15 files changed, 119 insertions(+), 109 deletions(-)
[impala] 04/04: IMPALA-12211: Adapt FIPS detection to handle OpenSSL 3
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c5ec833aec58425d203fbc452058ea837ad2ae75
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Jun 13 21:38:35 2023 -0700
IMPALA-12211: Adapt FIPS detection to handle OpenSSL 3
Various pieces of our code detect FIPS and enable/disable
code and tests accordingly (e.g. old hashing algorithms
like MD5 are not allowed). The detection logic currently
uses FIPS_mode(), but FIPS_mode() was removed in OpenSSL 3.
The OpenSSL 3 equivalent is EVP_default_properties_is_fips_enabled().
This adds a new IsFIPSMode() function in util/openssl-util.h
that uses OPENSSL_VERSION_NUMBER to dispatch to either
FIPS_mode() or EVP_default_properties_is_fips_enabled().
This also pulls in the Kudu changes to handle this for KRPC.
The original upstream Kudu commits were:
c24629083e : Follow-up on OpenSSL 3 FIPS_mode removal
acac73ecda : Fix OpenSSL3 FIPS_mode() issue on RHEL9.1
Testing:
- Built on Ubuntu 22 / Redhat 9 with OpenSSL 3
Change-Id: I78571c8086bea06bf7b355cfe86a16f190cde067
Reviewed-on: http://gerrit.cloudera.org:8080/20072
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Wenzhe Zhou <wz...@cloudera.com>
---
be/src/exprs/expr-test.cc | 10 +++++-----
be/src/exprs/mask-functions-ir.cc | 4 ++--
be/src/exprs/utility-functions-ir.cc | 9 ++++-----
be/src/kudu/util/openssl_util.cc | 6 +++++-
be/src/kudu/util/openssl_util.h | 4 ++++
be/src/util/openssl-util.h | 16 ++++++++++++----
be/src/util/webserver-test.cc | 6 +++---
be/src/util/webserver.cc | 3 ++-
8 files changed, 37 insertions(+), 21 deletions(-)
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index a0be31492..5edd34e65 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -29,7 +29,6 @@
#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>
-#include <openssl/crypto.h>
#include <openssl/sha.h>
#include "codegen/llvm-codegen.h"
@@ -71,6 +70,7 @@
#include "util/debug-util.h"
#include "util/decimal-util.h"
#include "util/metrics.h"
+#include "util/openssl-util.h"
#include "util/string-parser.h"
#include "util/string-util.h"
#include "util/test-info.h"
@@ -5706,7 +5706,7 @@ TEST_P(ExprTest, SHAFunctions) {
unsigned char sha384[SHA384_DIGEST_LENGTH];
unsigned char sha512[SHA512_DIGEST_LENGTH];
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
TestError(sha1fn);
TestError(sha2fn + ", 224)");
TestError(sha2fn + ", 256)");
@@ -5733,7 +5733,7 @@ TEST_P(ExprTest, SHAFunctions) {
TestStringValue(sha2fn + ", 512)", expected);
// Test empty strings. Empty string is valid input and produces hash.
- if (!FIPS_mode()) {
+ if (!IsFIPSMode()) {
SHA1(input, 0, sha1);
expected = ToHex(sha1, SHA_DIGEST_LENGTH);
TestStringValue("sha1('')", expected);
@@ -5764,7 +5764,7 @@ TEST_P(ExprTest, SHAFunctions) {
}
TEST_P(ExprTest, MD5Function) {
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
TestError("md5('foo')");
} else {
std::string expected("1fdf956bdad98101171512d18eec3bcf");
@@ -10787,7 +10787,7 @@ TEST_P(ExprTest, MaskTest) {
}
TEST_P(ExprTest, MaskHashTest) {
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
TestStringValue("mask_hash('TestString-123')",
"f3a58111be6ecec11449ac44654e72376b7759883ea11723b6e51354d50436de"
"645bd061cb5c2b07b68e15b7a7c342cac41f69b9c4efe19e810bbd7abf639a1c");
diff --git a/be/src/exprs/mask-functions-ir.cc b/be/src/exprs/mask-functions-ir.cc
index c96be187d..b6725ea5c 100644
--- a/be/src/exprs/mask-functions-ir.cc
+++ b/be/src/exprs/mask-functions-ir.cc
@@ -20,13 +20,13 @@
#include <boost/locale/generator.hpp>
#include <boost/locale/utf8_codecvt.hpp>
#include <gutil/strings/substitute.h>
-#include <openssl/crypto.h>
#include <openssl/err.h>
#include <openssl/sha.h>
#include "exprs/anyval-util.h"
#include "exprs/math-functions.h"
#include "exprs/string-functions.h"
+#include "util/openssl-util.h"
#include "util/ubsan.h"
#include "common/names.h"
@@ -943,7 +943,7 @@ TimestampVal MaskFunctions::Mask(FunctionContext* ctx, const TimestampVal& val,
StringVal MaskFunctions::MaskHash(FunctionContext* ctx, const StringVal& val) {
// Hive hash the value by sha256 and encoding it into a lower case hex string in
// non FIPS mode. In FIPS enabled mode, it's required to use sha512 for mask hash.
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
StringVal sha512_hash(ctx, SHA512_DIGEST_LENGTH);
if (UNLIKELY(sha512_hash.is_null)) return StringVal::null();
discard_result(SHA512(val.ptr, val.len, sha512_hash.ptr));
diff --git a/be/src/exprs/utility-functions-ir.cc b/be/src/exprs/utility-functions-ir.cc
index ce2925350..2d6ecfaec 100644
--- a/be/src/exprs/utility-functions-ir.cc
+++ b/be/src/exprs/utility-functions-ir.cc
@@ -22,13 +22,12 @@
#include "exprs/math-functions.h"
#include "exprs/string-functions.h"
-#include <openssl/crypto.h>
-#include <openssl/evp.h>
#include <openssl/md5.h>
#include <openssl/sha.h>
#include "runtime/runtime-state.h"
#include "udf/udf-internal.h"
#include "util/debug-util.h"
+#include "util/openssl-util.h"
#include "util/time.h"
#include "common/names.h"
@@ -247,7 +246,7 @@ StringVal UtilityFunctions::Sha1(FunctionContext* ctx, const StringVal& input_st
}
// SHA-1 is not supported for FIPS.
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
ctx->SetError("sha1 is not supported in FIPS mode.");
return StringVal::null();
} else {
@@ -266,7 +265,7 @@ StringVal UtilityFunctions::Sha2(FunctionContext* ctx, const StringVal& input_st
}
// SHA-224 and SHA-256 are deprecated for FIPS mode.
- if (FIPS_mode() && (bit_len.val == 224 || bit_len.val == 256)) {
+ if (IsFIPSMode() && (bit_len.val == 224 || bit_len.val == 256)) {
ctx->SetError("Only bit lengths 384 and 512 are supported in FIPS mode.");
return StringVal::null();
}
@@ -308,7 +307,7 @@ StringVal UtilityFunctions::Md5(FunctionContext* ctx, const StringVal& input_str
if (input_str.is_null) {
return StringVal::null();
}
- if (UNLIKELY(FIPS_mode())) {
+ if (UNLIKELY(IsFIPSMode())) {
ctx->SetError("MD5 is not supported in FIPS mode.");
return StringVal::null();
} else {
diff --git a/be/src/kudu/util/openssl_util.cc b/be/src/kudu/util/openssl_util.cc
index b210f3d66..33e985b62 100644
--- a/be/src/kudu/util/openssl_util.cc
+++ b/be/src/kudu/util/openssl_util.cc
@@ -95,7 +95,11 @@ void ThreadIdCB(CRYPTO_THREADID* tid) {
#endif
void CheckFIPSMode() {
- auto fips_mode = FIPS_mode();
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
+ int fips_mode = FIPS_mode();
+#else
+ int fips_mode = EVP_default_properties_is_fips_enabled(NULL);
+#endif
// If the environment variable KUDU_REQUIRE_FIPS_MODE is set to "1", we
// check if FIPS approved mode is enabled. If not, we crash the process.
// As this is used in clients as well, we can't use gflags to set this.
diff --git a/be/src/kudu/util/openssl_util.h b/be/src/kudu/util/openssl_util.h
index d1e4b16a3..b62eca59a 100644
--- a/be/src/kudu/util/openssl_util.h
+++ b/be/src/kudu/util/openssl_util.h
@@ -17,9 +17,13 @@
#pragma once
+#include <openssl/crypto.h>
#include <openssl/err.h>
#include <openssl/pem.h>
#include <openssl/ssl.h>
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/types.h>
+#endif
#include <openssl/x509.h>
#include <functional>
diff --git a/be/src/util/openssl-util.h b/be/src/util/openssl-util.h
index c7a30e400..0864e5fb6 100644
--- a/be/src/util/openssl-util.h
+++ b/be/src/util/openssl-util.h
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-#ifndef IMPALA_UTIL_OPENSSL_UTIL_H
-#define IMPALA_UTIL_OPENSSL_UTIL_H
+#pragma once
#include <openssl/aes.h>
+#include <openssl/crypto.h>
#include <openssl/evp.h>
#include <openssl/sha.h>
#include <openssl/ssl.h>
@@ -59,6 +59,16 @@ bool IsExternalTlsConfigured();
/// and again periodically to add new entropy.
void SeedOpenSSLRNG();
+/// Returns true if OpenSSL's FIPS mode is enabled. This calculation varies by the OpenSSL
+/// version, so it is useful to have a shared function.
+inline bool IsFIPSMode() {
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
+ return FIPS_mode() > 0;
+#else
+ return EVP_default_properties_is_fips_enabled(nullptr) > 0;
+#endif
+};
+
enum AES_CIPHER_MODE {
AES_256_CFB,
AES_256_CTR,
@@ -200,5 +210,3 @@ class EncryptionKey {
};
}
-
-#endif
diff --git a/be/src/util/webserver-test.cc b/be/src/util/webserver-test.cc
index 29256cbea..2ed6d567f 100644
--- a/be/src/util/webserver-test.cc
+++ b/be/src/util/webserver-test.cc
@@ -24,7 +24,6 @@
#include <boost/lexical_cast.hpp>
#include <gutil/strings/substitute.h>
#include <map>
-#include <openssl/crypto.h>
#include <openssl/ssl.h>
#include <regex>
@@ -35,6 +34,7 @@
#include "util/default-path-handlers.h"
#include "util/kudu-status-util.h"
#include "util/metrics.h"
+#include "util/openssl-util.h"
#include "util/os-util.h"
#include "util/webserver.h"
@@ -662,7 +662,7 @@ TEST(Webserver, StartWithPasswordFileTest) {
MetricGroup metrics("webserver-test");
Webserver webserver("", FLAGS_webserver_port, &metrics);
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
ASSERT_FALSE(webserver.Start().ok());
} else {
ASSERT_OK(webserver.Start());
@@ -701,7 +701,7 @@ TEST(Webserver, StartWithPasswordFileTest) {
}
TEST(Webserver, StartWithMissingPasswordFileTest) {
- if (FIPS_mode()) return;
+ if (IsFIPSMode()) return;
stringstream password_file;
password_file << getenv("IMPALA_HOME") << "/be/src/testutil/doesntexist";
auto password =
diff --git a/be/src/util/webserver.cc b/be/src/util/webserver.cc
index d5a754911..a4436f6f8 100644
--- a/be/src/util/webserver.cc
+++ b/be/src/util/webserver.cc
@@ -55,6 +55,7 @@
#include "util/jwt-util.h"
#include "util/mem-info.h"
#include "util/metrics.h"
+#include "util/openssl-util.h"
#include "util/os-info.h"
#include "util/os-util.h"
#include "util/pretty-printer.h"
@@ -450,7 +451,7 @@ Status Webserver::Start() {
}
if (!FLAGS_webserver_password_file.empty()) {
- if (FIPS_mode()) {
+ if (IsFIPSMode()) {
return Status("HTTP digest authorization is not supported in FIPS approved mode.");
} else {
// Squeasel doesn't log anything if it can't stat the password file (but will if it
[impala] 02/04: IMPALA-12179 (part 2): Use /etc/os-release for bootstrap_toolchain.py
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit f0065cdb7070276650c41d84fe1d061d87069d83
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Jun 13 20:03:59 2023 -0700
IMPALA-12179 (part 2): Use /etc/os-release for bootstrap_toolchain.py
This removes the usage of lsb_release in bootstrap_toolchain.py
in favor of reading /etc/os-release. /etc/os-release is available
on all distributions that we support. This combines the ID
and the major version component of VERSION_ID to produce the
distribution identifier. A few example:
Ubuntu 16.04: ID=ubuntu, VERSION_ID=16.04 => ubuntu16
Rocky 8.5: ID=rocky, VERSION_ID=8.5 => rocky8
SLES 15.1: ID=sles, VERSION_ID=15.1 => sles15
As cleanup, this removes old distributions that we no longer
support (e.g. Redhat 5/6, Debian, Sles 11, etc). It also removes
the unused CDH component of the OS_MAPPING.
The values used in OS_MAPPING are based on the database of
/etc/os-release files available at https://github.com/chef/os_release
Testing:
- Ran the logic against the /etc/os-release files for Redhat/Centos 7,
Redhat/Centos/Rocky/Almalinux 8, Ubuntu, and SLES 12/15.
Change-Id: Ida3ffb8525c5b750ddbf9fd3ed5d0782fac9cdd0
Reviewed-on: http://gerrit.cloudera.org:8080/20070
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/bootstrap_toolchain.py | 97 ++++++++++++++++++++++------------------------
1 file changed, 47 insertions(+), 50 deletions(-)
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 59628a3c6..7d9cc70b6 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -56,12 +56,10 @@
from __future__ import absolute_import, division, print_function
import logging
-import glob
import multiprocessing.pool
import os
import platform
import random
-import re
import shutil
import subprocess
import sys
@@ -71,36 +69,22 @@ import time
from collections import namedtuple
from string import Template
-# Maps return values from 'lsb_release -irs' to the corresponding OS labels for both the
-# toolchain and the CDP components.
-OsMapping = namedtuple('OsMapping', ['lsb_release', 'toolchain', 'cdh'])
+# Maps the ID + major version of the VERSION_ID from /etc/os-release to the corresponding
+# OS label for the toolchain. See https://github.com/chef/os_release for a database of
+# /etc/os-release files.
+OsMapping = namedtuple('OsMapping', ['release', 'toolchain'])
OS_MAPPING = [
- OsMapping("centos5", "ec2-package-centos-5", None),
- OsMapping("centos6", "ec2-package-centos-6", "redhat6"),
- OsMapping("centos7", "ec2-package-centos-7", "redhat7"),
- OsMapping("centos8", "ec2-package-centos-8", "redhat8"),
- OsMapping("rocky8", "ec2-package-centos-8", "redhat8"),
- OsMapping("almalinux8", "ec2-package-centos-8", "redhat8"),
- OsMapping("redhatenterpriseserver5", "ec2-package-centos-5", None),
- OsMapping("redhatenterpriseserver6", "ec2-package-centos-6", "redhat6"),
- OsMapping("redhatenterpriseserver7", "ec2-package-centos-7", "redhat7"),
- OsMapping("redhatenterprise8", "ec2-package-centos-8", "redhat8"),
- OsMapping("redhatenterpriseserver8", "ec2-package-centos-8", "redhat8"),
- OsMapping("debian6", "ec2-package-debian-6", None),
- OsMapping("debian7", "ec2-package-debian-7", None),
- OsMapping("debian8", "ec2-package-debian-8", "debian8"),
- OsMapping("suselinux11", "ec2-package-sles-11", None),
- OsMapping("suselinux12", "ec2-package-sles-12", "sles12"),
- OsMapping("suse12", "ec2-package-sles-12", "sles12"),
- OsMapping("suselinux15", "ec2-package-sles-15", "sles15"),
- OsMapping("suse15", "ec2-package-sles-15", "sles15"),
- OsMapping("ubuntu12.04", "ec2-package-ubuntu-12-04", None),
- OsMapping("ubuntu14.04", "ec2-package-ubuntu-14-04", None),
- OsMapping("ubuntu15.04", "ec2-package-ubuntu-14-04", None),
- OsMapping("ubuntu15.10", "ec2-package-ubuntu-14-04", None),
- OsMapping('ubuntu16.04', "ec2-package-ubuntu-16-04", "ubuntu1604"),
- OsMapping('ubuntu18.04', "ec2-package-ubuntu-18-04", "ubuntu1804"),
- OsMapping('ubuntu20.04', "ec2-package-ubuntu-20-04", "ubuntu2004")
+ OsMapping("rhel7", "ec2-package-centos-7"),
+ OsMapping("centos7", "ec2-package-centos-7"),
+ OsMapping("rhel8", "ec2-package-centos-8"),
+ OsMapping("centos8", "ec2-package-centos-8"),
+ OsMapping("rocky8", "ec2-package-centos-8"),
+ OsMapping("almalinux8", "ec2-package-centos-8"),
+ OsMapping("sles12", "ec2-package-sles-12"),
+ OsMapping("sles15", "ec2-package-sles-15"),
+ OsMapping('ubuntu16', "ec2-package-ubuntu-16-04"),
+ OsMapping('ubuntu18', "ec2-package-ubuntu-18-04"),
+ OsMapping('ubuntu20', "ec2-package-ubuntu-20-04")
]
@@ -350,6 +334,7 @@ class ApacheComponent(EnvVersionedPackage):
unpack_directory_tmpl=unpack_directory_tmpl,
makedir=makedir, template_subs_in=template_subs)
+
class ToolchainKudu(ToolchainPackage):
def __init__(self, platform_label=None):
super(ToolchainKudu, self).__init__('kudu', platform_release=platform_label)
@@ -377,32 +362,45 @@ def try_get_platform_release_label():
return None
-# Cache "lsb_release -irs" to avoid excessive logging from sh, and
-# to shave a little bit of time.
-lsb_release_cache = None
+# Cache the /etc/os-release calculation to shave a little bit of time.
+os_release_cache = None
def get_platform_release_label(release=None):
"""Gets the right package label from the OS version. Raise exception if not found.
- 'release' can be provided to override the underlying OS version.
+ 'release' can be provided to override the underlying OS version. This uses
+ ID and VERSION_ID from /etc/os-release to identify a distribution. Specifically,
+ this returns the concatenation of the ID and the major version component
+ of VERSION_ID. i.e. ID=ubuntu VERSION_ID=16.04 => ubuntu16
"""
- global lsb_release_cache
+ global os_release_cache
if not release:
- if lsb_release_cache:
- release = lsb_release_cache
+ if os_release_cache:
+ release = os_release_cache
else:
- lsb_release = subprocess.check_output(
- ["lsb_release", "-irs"], universal_newlines=True)
- release = "".join([x.lower() for x in lsb_release.split()])
- # Only need to check against the major release if RHEL, CentOS or Suse
- for distro in ['centos', 'rocky', 'almalinux', 'redhatenterprise',
- 'redhatenterpriseserver', 'suse']:
- if distro in release:
- release = release.split('.')[0]
- break
- lsb_release_cache = release
+ os_id = None
+ os_major_version = None
+ with open("/etc/os-release") as f:
+ for line in f:
+ # We assume that ID and VERSION_ID are present and don't contain '=' inside
+ # the actual value. This is true for all distributions we currently support.
+ if line.startswith("ID="):
+ os_id = line.split("=")[1].strip().strip('"')
+ elif line.startswith("VERSION_ID="):
+ os_version_id = line.split("=")[1].strip().strip('"')
+ # Some distributions have a major version that doesn't change (e.g. 3.12.0
+ # and 3.12.0). The distributions that we support don't do this. This
+ # calculation would need to change for that circumstance.
+ os_major_version = os_version_id.split(".")[0]
+
+ if os_id is None or os_major_version is None:
+ raise Exception("Error parsing /etc/os-release: "
+ "os_id={0} os_major_version={1}".format(os_id, os_major_version))
+
+ release = "{0}{1}".format(os_id, os_major_version)
+ os_release_cache = release
for mapping in OS_MAPPING:
- if re.search(mapping.lsb_release, release):
+ if mapping.release == release:
return mapping
raise Exception("Could not find package label for OS version: {0}.".format(release))
@@ -563,7 +561,6 @@ def main():
downloads = []
if os.getenv("SKIP_TOOLCHAIN_BOOTSTRAP", "false") != "true":
downloads += get_toolchain_downloads()
- kudu_download = None
if os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true":
create_directory_from_env_var("CDP_COMPONENTS_HOME")
create_directory_from_env_var("APACHE_COMPONENTS_HOME")
[impala] 01/04: IMPALA-12179 (part 1): Remove dependency on lsb_release for docker CMake
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9c6df6a69142801467c82fa8e0d972b366766f99
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Jun 13 19:12:20 2023 -0700
IMPALA-12179 (part 1): Remove dependency on lsb_release for docker CMake
Newer operating systems like Redhat 9 do not supply
lsb_release as an official package. The /etc/os-release
file provides the same information in a more convenient
form. CMake 3.22 added support for reading those
/etc/os-release values directly via cmake_host_system_information().
This changes docker/CMakeLists.txt to use the new CMake
cmake_host_system_information() APIs to get values from
/etc/os-release. This removes the lsb_release code.
Testing:
- Ran a docker build locally and verified it detected
the distribution / version correctly
Change-Id: I04afd2b1c923f1331f7234d53a105a17956e3e18
Reviewed-on: http://gerrit.cloudera.org:8080/20069
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
CMakeLists.txt | 2 +-
docker/CMakeLists.txt | 47 ++++++++++++++++++++++-------------------------
2 files changed, 23 insertions(+), 26 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7b86e421d..8f91ae26a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.22)
# This is a Kudu-specific flag that disables Kudu targets that are test-only.
set(NO_TESTS 1)
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index 7f0455760..b4aebf352 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -25,42 +25,39 @@ set(IMPALA_UTILITY_BUILD_CONTEXT_DIR
set(DOCKER_BUILD ${CMAKE_SOURCE_DIR}/docker/docker-build.sh)
-find_program(LSB_RELEASE_EXEC lsb_release)
-execute_process(COMMAND ${LSB_RELEASE_EXEC} -is
- OUTPUT_VARIABLE LSB_RELEASE_ID
- OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs
- OUTPUT_VARIABLE LSB_RELEASE_VERSION
- OUTPUT_STRIP_TRAILING_WHITESPACE
-)
+# These CMake commands are using information from /etc/os-release
+cmake_host_system_information(RESULT OS_DISTRIB_ID QUERY DISTRIB_ID)
+cmake_host_system_information(RESULT OS_DISTRIB_VERSION_ID QUERY DISTRIB_VERSION_ID)
set(QUICKSTART_BASE_IMAGE "UNSUPPORTED")
set(DISTRO_BASE_IMAGE "UNSUPPORTED")
-if(${LSB_RELEASE_ID} STREQUAL "Ubuntu")
- if(${LSB_RELEASE_VERSION} STREQUAL "16.04" OR
- ${LSB_RELEASE_VERSION} STREQUAL "18.04" OR
- ${LSB_RELEASE_VERSION} STREQUAL "20.04")
- set(DISTRO_BASE_IMAGE "ubuntu:${LSB_RELEASE_VERSION}")
- set(QUICKSTART_BASE_IMAGE "ubuntu:${LSB_RELEASE_VERSION}")
+# The CMake variables are using information from /etc/os-release.
+# A database of /etc/os-release files is available at
+# https://github.com/chef/os_release
+# These comparisons are based on those values.
+if(${OS_DISTRIB_ID} STREQUAL "ubuntu")
+ if(${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04")
+ set(DISTRO_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
+ set(QUICKSTART_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
endif()
- if (${LSB_RELEASE_VERSION} STREQUAL "16.04" OR
- ${LSB_RELEASE_VERSION} STREQUAL "18.04")
+ if (${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04")
set(PIP "python-pip")
- elseif (${LSB_RELEASE_VERSION} STREQUAL "20.04")
+ elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04")
set(PIP "python3-pip")
endif()
-elseif(${LSB_RELEASE_ID} STREQUAL "RedHatEnterpriseServer" OR
- ${LSB_RELEASE_ID} STREQUAL "RedHatEnterprise" OR
- ${LSB_RELEASE_ID} STREQUAL "Rocky" OR
- ${LSB_RELEASE_ID} STREQUAL "AlmaLinux" OR
- ${LSB_RELEASE_ID} STREQUAL "CentOS")
+elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR
+ ${OS_DISTRIB_ID} STREQUAL "rocky" OR
+ ${OS_DISTRIB_ID} STREQUAL "almalinux" OR
+ ${OS_DISTRIB_ID} STREQUAL "centos")
# The Quickstart images currently don't support using a Redhat
# base image, so this doesn't set QUICKSTART_BASE_IMAGE.
- if(${LSB_RELEASE_VERSION} MATCHES "7.*")
+ if(${OS_DISTRIB_VERSION_ID} MATCHES "7.*")
set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT7_DOCKER_BASE}")
- elseif(${LSB_RELEASE_VERSION} MATCHES "8.*")
+ elseif(${OS_DISTRIB_VERSION_ID} MATCHES "8.*")
set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT8_DOCKER_BASE}")
endif()
endif()
[impala] 03/04: IMPALA-12179 (part 3): Remove remaining lsb_release references
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6222785ef57dc23626be01e5db03c274e2413ea1
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Jun 13 20:42:38 2023 -0700
IMPALA-12179 (part 3): Remove remaining lsb_release references
This removes a few stray lsb_release references in distcc
scripts and the install_docker.sh script. It then removes
the redhat-lsb package from the list of installed packages.
Testing:
- Ran a build on Rocky 8.5
- Ran dockerised tests on Ubuntu 20
Change-Id: I9d84e9ab8076fd8cc4727a5da118d9a747d4a005
Reviewed-on: http://gerrit.cloudera.org:8080/20071
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/bootstrap_system.sh | 2 +-
bin/distcc/distcc_server_bootstrap.sh | 8 ++++----
bin/distcc/distcc_server_setup.sh | 12 ++++++------
bin/jenkins/install_docker.sh | 2 +-
4 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index 04d02e6bb..d470e9036 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -245,7 +245,7 @@ redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \
cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \
postgresql postgresql-server \
wget vim-common nscd cmake fuse-devel zlib-devel \
- psmisc lsof openssh-server redhat-lsb python3-devel python3-setuptools \
+ psmisc lsof openssh-server python3-devel python3-setuptools \
net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \
java-${REDHAT_JAVA_VERSION}-openjdk-src java-${REDHAT_JAVA_VERSION}-openjdk-devel
diff --git a/bin/distcc/distcc_server_bootstrap.sh b/bin/distcc/distcc_server_bootstrap.sh
index fd69a560e..187366478 100755
--- a/bin/distcc/distcc_server_bootstrap.sh
+++ b/bin/distcc/distcc_server_bootstrap.sh
@@ -33,10 +33,10 @@ if [[ "$USER" != "root" ]]; then
exit 1
fi
-LSB_ID=$(lsb_release -is)
-LSB_VERSION=$(lsb_release -rs)
-if [[ "$LSB_ID" == Ubuntu ]]; then
- if ! [[ $LSB_VERSION == 16.04 || $LSB_VERSION == 18.04 ]]; then
+OS_ID=$(source /etc/os-release && echo $ID)
+OS_VERSION=$(source /etc/os-release && echo $VERSION_ID)
+if [[ "$OS_ID" == Ubuntu ]]; then
+ if ! [[ $OS_VERSION == 16.04 || $OS_VERSION == 18.04 ]]; then
echo "This script only supports Ubuntu 16.04 and 18.04" >&2
exit 1
fi
diff --git a/bin/distcc/distcc_server_setup.sh b/bin/distcc/distcc_server_setup.sh
index 38a93dce0..8d5d7bb98 100755
--- a/bin/distcc/distcc_server_setup.sh
+++ b/bin/distcc/distcc_server_setup.sh
@@ -57,18 +57,18 @@ fi
set -x
ALLOWED_NETS=$1
-LSB_ID=$(lsb_release -is)
-LSB_VERSION=$(lsb_release -rs)
-if [[ "$LSB_ID" == Ubuntu ]]; then
- if ! [[ $LSB_VERSION == 14.04 || $LSB_VERSION == 16.04 || $LSB_VERSION == 18.04 ]]; then
+OS_ID=$(source /etc/os-release && echo "$ID")
+OS_VERSION=$(source /etc/os-release && echo "$VERSION_ID")
+if [[ "$OS_ID" == ubuntu ]]; then
+ if ! [[ $OS_VERSION == 14.04 || $OS_VERSION == 16.04 || $OS_VERSION == 18.04 ]]; then
echo "This script only supports Ubuntu 14.04, 16.04 and 18.04" >&2
exit 1
fi
LINUX_FLAVOUR=ubuntu
DISTCCD_USER=distccd
DISTCCD_SERVICE=distcc
-elif [[ "$LSB_ID" == CentOS ]]; then
- if ! [[ $LSB_VERSION == 6.* || $LSB_VERSION = 7.* ]]; then
+elif [[ "$OS_ID" == centos ]]; then
+ if ! [[ $OS_VERSION == 6.* || $OS_VERSION = 7.* ]]; then
echo "This script only supports CentOS 6 and 7" >&2
exit 1
fi
diff --git a/bin/jenkins/install_docker.sh b/bin/jenkins/install_docker.sh
index 261663822..49c4b9f6e 100755
--- a/bin/jenkins/install_docker.sh
+++ b/bin/jenkins/install_docker.sh
@@ -63,7 +63,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
grep '9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88'
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
- $(lsb_release -cs) stable"
+ $(source /etc/os-release && echo $UBUNTU_CODENAME) stable"
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
sudo service docker restart