You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/04/26 06:23:38 UTC

[impala] 01/04: Make infra/python compatible with both Python 2 & 3

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit e4388740f765275a4fbd004adc67f1fcc13403a7
Author: Akshesh <ak...@gmail.com>
AuthorDate: Tue Feb 26 21:44:46 2019 +0700

    Make infra/python compatible with both Python 2 & 3
    
    Change-Id: If4285a021bb581f88425daa52ef8a3f844017d82
    Reviewed-on: http://gerrit.cloudera.org:8080/13070
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Reviewed-by: Akshesh Doshi <ak...@gmail.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 infra/python/bootstrap_virtualenv.py |  7 ++++---
 infra/python/deps/find_py26.py       |  4 +++-
 infra/python/deps/pip_download.py    | 29 ++++++++++++++++++-----------
 3 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index 305e588..27c527f 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -34,6 +34,7 @@
 # system. If the default 'python' command refers to < 2.6, python 2.6 will be used
 # instead.
 
+from __future__ import print_function
 import glob
 import logging
 import optparse
@@ -239,7 +240,7 @@ def install_compiled_deps_if_possible():
   # problem with inline declarations in older libc headers. Setting -fgnu89-inline is a
   # workaround.
   distro_version = ''.join(exec_cmd(["lsb_release", "-irs"]).lower().split())
-  print distro_version
+  print(distro_version)
   if distro_version.startswith("centos5."):
     env["CFLAGS"] = "-fgnu89-inline"
 
@@ -381,8 +382,8 @@ if __name__ == "__main__":
 
   if options.print_ld_library_path:
     kudu_client_dir = find_kudu_client_install_dir()
-    print os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
-                                os.path.join(kudu_client_dir, 'lib64')])
+    print(os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
+                                    os.path.join(kudu_client_dir, 'lib64')]))
     sys.exit()
 
   logging.basicConfig(level=getattr(logging, options.log_level))
diff --git a/infra/python/deps/find_py26.py b/infra/python/deps/find_py26.py
index 9135469..dc628c7 100755
--- a/infra/python/deps/find_py26.py
+++ b/infra/python/deps/find_py26.py
@@ -21,6 +21,7 @@
 # system command to stdout. The script exits with a nonzero exit code if
 # Python 2.6+ is not present.
 
+from __future__ import print_function
 import os
 import subprocess
 import sys
@@ -42,4 +43,5 @@ def detect_python_cmd():
         return cmd_path
   raise Exception("Could not find minimum required python version 2.6")
 
-print detect_python_cmd()
+
+print(detect_python_cmd())
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index 3e593c4..354bff4 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -21,6 +21,7 @@
 # over which archive type is downloaded and what post-download steps are executed.
 # This script requires Python 2.6+.
 
+from __future__ import print_function
 import hashlib
 import multiprocessing.pool
 import os
@@ -29,7 +30,13 @@ import re
 import sys
 from random import randint
 from time import sleep
-from urllib import urlopen, FancyURLopener
+# The path to import is different for libraries in Python 2 & 3 - this try/catch ensures
+# that this code runs in both
+try:
+  # This should be removed when support for Python2 is dropped
+  from urllib import urlopen, FancyURLopener
+except ImportError:
+  from urllib.request import urlopen, FancyURLopener
 
 NUM_DOWNLOAD_ATTEMPTS = 8
 
@@ -48,7 +55,7 @@ def check_digest(filename, algorithm, expected_digest):
     # Fallback to hardcoded set if hashlib.algorithms_available doesn't exist.
     supported_algorithms = set(['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'])
   if algorithm not in supported_algorithms:
-    print 'Hash algorithm {0} is not supported by hashlib'.format(algorithm)
+    print('Hash algorithm {0} is not supported by hashlib'.format(algorithm))
     return False
   h = hashlib.new(algorithm)
   h.update(open(filename).read())
@@ -60,18 +67,18 @@ def retry(func):
   '''Retry decorator.'''
 
   def wrapper(*args, **kwargs):
-    for try_num in xrange(NUM_DOWNLOAD_ATTEMPTS):
+    for try_num in range(NUM_DOWNLOAD_ATTEMPTS):
       if try_num > 0:
         sleep_len = randint(5, 10 * 2 ** try_num)
-        print 'Sleeping for {0} seconds before retrying'.format(sleep_len)
+        print('Sleeping for {0} seconds before retrying'.format(sleep_len))
         sleep(sleep_len)
       try:
         result = func(*args, **kwargs)
         if result:
           return result
       except Exception as e:
-        print e
-    print 'Download failed after several attempts.'
+        print(e)
+    print('Download failed after several attempts.')
     sys.exit(1)
 
   return wrapper
@@ -83,7 +90,7 @@ def get_package_info(pkg_name, pkg_version):
   # same result is always returned even if the ordering changed on the server.
   candidates = []
   url = '{0}/simple/{1}/'.format(PYPI_MIRROR, pkg_name)
-  print 'Getting package info from {0}'.format(url)
+  print('Getting package info from {0}'.format(url))
   # The web page should be in PEP 503 format (https://www.python.org/dev/peps/pep-0503/).
   # We parse the page with regex instead of an html parser because that requires
   # downloading an extra package before running this script. Since the HTML is guaranteed
@@ -101,7 +108,7 @@ def get_package_info(pkg_name, pkg_version):
         file_name.endswith('-{0}.zip'.format(pkg_version))):
       candidates.append((file_name, path, hash_algorithm, digest))
   if not candidates:
-    print 'Could not find archive to download for {0} {1}'.format(pkg_name, pkg_version)
+    print('Could not find archive to download for {0} {1}'.format(pkg_name, pkg_version))
     return (None, None, None, None)
   return sorted(candidates)[0]
 
@@ -113,16 +120,16 @@ def download_package(pkg_name, pkg_version):
     return False
   if os.path.isfile(file_name) and check_digest(file_name, hash_algorithm,
       expected_digest):
-    print 'File with matching digest already exists, skipping {0}'.format(file_name)
+    print('File with matching digest already exists, skipping {0}'.format(file_name))
     return True
   downloader = FancyURLopener()
   pkg_url = '{0}/packages/{1}'.format(PYPI_MIRROR, path)
-  print 'Downloading {0} from {1}'.format(file_name, pkg_url)
+  print('Downloading {0} from {1}'.format(file_name, pkg_url))
   downloader.retrieve(pkg_url, file_name)
   if check_digest(file_name, hash_algorithm, expected_digest):
     return True
   else:
-    print 'Hash digest check failed in file {0}.'.format(file_name)
+    print('Hash digest check failed in file {0}.'.format(file_name))
     return False
 
 def main():