[impala] 02/04: IMPALA-9627: Update utility scripts for Python 3

[bo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 0c72c98f91d291c02e281ebbd04ba94401e93c07
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Feb 28 09:43:22 2023 -0800

    IMPALA-9627: Update utility scripts for Python 3
    
    Updates utility scripts that don't use impala-python to work with Python
    3 so we can build on systems that don't include Python 2 (such as SLES
    15 SP4).
    
    Primarily adds 'universal_newlines=True' to subprocess calls so they
    return text rather than binary data in Python 3 with a change that's
    compatible with Python 2.
    
    Testing:
    - built in SLES 15 SP4 container with Python 3
    
    Change-Id: I7f4ce71fa1183aaeeca55d0666aeb113640c5cf2
    Reviewed-on: http://gerrit.cloudera.org:8080/19559
    Reviewed-by: Michael Smith <mi...@cloudera.com>
    Tested-by: Michael Smith <mi...@cloudera.com>
---
 bin/bootstrap_toolchain.py           | 6 ++++--
 infra/python/bootstrap_virtualenv.py | 9 ++++++---
 infra/python/deps/pip_download.py    | 5 +++--
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index e74e0b5f3..98ec9d7d6 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -107,7 +107,8 @@ def check_output(cmd_args):
      a non-zero return code. Similar to subprocess.check_output() which is only provided
      in python 2.7.
   """
-  process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+      universal_newlines=True)
   stdout, _ = process.communicate()
   if process.wait() != 0:
     raise Exception("Command with args '%s' failed with exit code %s:\n%s"
@@ -422,7 +423,8 @@ def check_output(cmd_args):
      a non-zero return code. Similar to subprocess.check_output() which is only provided
      in python 2.7.
   """
-  process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+      universal_newlines=True)
   stdout, _ = process.communicate()
   if process.wait() != 0:
     raise Exception("Command with args '%s' failed with exit code %s:\n%s"
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index d292e17cd..c3bc59932 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -42,7 +42,10 @@ import subprocess
 import sys
 import tarfile
 import tempfile
-import urllib
+try:
+  from urllib.request import pathname2url
+except ImportError:
+  from urllib import pathname2url
 from bootstrap_toolchain import ToolchainPackage
 
 LOG = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
@@ -124,7 +127,7 @@ def exec_cmd(args, **kwargs):
      'args' and 'kwargs' use the same format as subprocess.Popen().
   '''
   process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-      **kwargs)
+      universal_newlines=True, **kwargs)
   output = process.communicate()[0]
   if process.returncode != 0:
     raise Exception("Command returned non-zero status\nCommand: %s\nOutput: %s"
@@ -189,7 +192,7 @@ def exec_pip_install(args, cc="no-cc-available", env=None):
     third_party_pkg_install_cmd.append("--no-index")
 
   third_party_pkg_install_cmd.extend(["--find-links",
-      "file://%s" % urllib.pathname2url(os.path.abspath(DEPS_DIR))])
+      "file://%s" % pathname2url(os.path.abspath(DEPS_DIR))])
   third_party_pkg_install_cmd.extend(args)
   exec_cmd(third_party_pkg_install_cmd, env=env)
 
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index d56e0280e..9c41135d8 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -51,7 +51,7 @@ def check_digest(filename, algorithm, expected_digest):
     print('Hash algorithm {0} is not supported by hashlib'.format(algorithm))
     return False
   h = hashlib.new(algorithm)
-  h.update(open(filename).read())
+  h.update(open(filename, mode='rb').read())
   actual_digest = h.hexdigest()
   return actual_digest == expected_digest
 
@@ -89,7 +89,8 @@ def get_package_info(pkg_name, pkg_version):
   # We parse the page with regex instead of an html parser because that requires
   # downloading an extra package before running this script. Since the HTML is guaranteed
   # to be formatted according to PEP 503, this is acceptable.
-  pkg_info = subprocess.check_output(["wget", "-q", "-O", "-", url])
+  pkg_info = subprocess.check_output(
+      ["wget", "-q", "-O", "-", url], universal_newlines=True)
   regex = r'<a .*?href=\".*?packages/(.*?)#(.*?)=(.*?)\".*?>(.*?)<\/a>'
   for match in re.finditer(regex, pkg_info):
     path = match.group(1)