[impala] 02/03: IMPALA-9668: Obey SKIP_TOOLCHAIN_BOOTSTRAP during virtualenv bootstrap

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit b921d982b530d4546efeb10ceb02c82297859da9
Author: Laszlo Gaal <la...@cloudera.com>
AuthorDate: Mon Apr 20 14:39:47 2020 +0200

    IMPALA-9668: Obey SKIP_TOOLCHAIN_BOOTSTRAP during virtualenv bootstrap
    
    IMPALA-9626 broke the use case where the toolchain binaries are not
    downloaded from the native-toolchain S3 bucket, because
    SKIP_TOOLCHAIN_BOOTSTRAP is set to true.
    
    Fix this use case by checking SKIP_TOOLCHAIN_BOOTSTRAP in
    bin/bootstrap_environment.py:
    - if true: just check if the specified version of the Python binary is
      present at the expected toolchain location. If it is there, use it,
      otherwise throw an exception and abort the bootstrap process.
    - in any other case: proceed to download the Python binary as in
      bootstrap_toolchain.py.
    
    Test:
    - simulate the custom toolchain setup by downloading the toolchain
      binaries from the S3 bucket, copying them to a separate directory,
      symlinking them into Impala/toolchain, then executing buildall.sh
      with SKIP_BOOTSTRAP_TOOLCHAIN set to "true".
    
    Change-Id: Ic51b3c327b3cebc08edff90de931d07e35e0c319
    Reviewed-on: http://gerrit.cloudera.org:8080/15759
    Reviewed-by: Laszlo Gaal <la...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 bin/bootstrap_toolchain.py           | 2 +-
 infra/python/bootstrap_virtualenv.py | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 82362de..4720727 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -611,7 +611,7 @@ def get_toolchain_downloads():
   toolchain_packages += map(ToolchainPackage,
       ["avro", "binutils", "boost", "breakpad", "bzip2", "cctz", "cmake", "crcutil",
        "flatbuffers", "gdb", "gflags", "glog", "gperftools", "gtest", "libev",
-       "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf",
+       "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf", "python",
        "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib", "zstd"])
   toolchain_packages += [ToolchainPackage("thrift",
       explicit_version=os.environ.get("IMPALA_THRIFT11_VERSION"))]
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index cccdfe0..4a8dff1 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -50,6 +50,8 @@ from bootstrap_toolchain import ToolchainPackage
 
 LOG = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
 
+SKIP_TOOLCHAIN_BOOTSTRAP = "SKIP_TOOLCHAIN_BOOTSTRAP"
+
 DEPS_DIR = os.path.join(os.path.dirname(__file__), "deps")
 ENV_DIR = os.path.join(os.path.dirname(__file__), "env")
 
@@ -192,7 +194,9 @@ def find_file(*paths):
 
 def download_toolchain_python():
   '''Grabs the Python implementation from the Impala toolchain, using the machinery from
-     bin/bootstrap_toolchain.py
+     bin/bootstrap_toolchain.py.
+     Skip the download if SKIP_TOOLCHAIN_BOOTSTRAP=true in the environment. In that case
+     only the presence of the Python executable is checked in the toolchain location.
   '''
 
   toolchain_root = os.environ.get("IMPALA_TOOLCHAIN")
@@ -201,7 +205,8 @@ def download_toolchain_python():
         "Impala environment not set up correctly, make sure $IMPALA_TOOLCHAIN is set.")
 
   package = ToolchainPackage("python")
-  package.download()
+  if not (os.environ.get(SKIP_TOOLCHAIN_BOOTSTRAP) == 'true'):
+    package.download()
   python_cmd = os.path.join(package.pkg_directory(), "bin/python")
   if not os.path.exists(python_cmd):
     raise Exception("Unexpected error bootstrapping python from toolchain: {0} does not "