You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/04/26 06:23:38 UTC
[impala] 01/04: Make infra/python compatible with both Python 2 & 3
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e4388740f765275a4fbd004adc67f1fcc13403a7
Author: Akshesh <ak...@gmail.com>
AuthorDate: Tue Feb 26 21:44:46 2019 +0700
Make infra/python compatible with both Python 2 & 3
Change-Id: If4285a021bb581f88425daa52ef8a3f844017d82
Reviewed-on: http://gerrit.cloudera.org:8080/13070
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Akshesh Doshi <ak...@gmail.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
infra/python/bootstrap_virtualenv.py | 7 ++++---
infra/python/deps/find_py26.py | 4 +++-
infra/python/deps/pip_download.py | 29 ++++++++++++++++++-----------
3 files changed, 25 insertions(+), 15 deletions(-)
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index 305e588..27c527f 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -34,6 +34,7 @@
# system. If the default 'python' command refers to < 2.6, python 2.6 will be used
# instead.
+from __future__ import print_function
import glob
import logging
import optparse
@@ -239,7 +240,7 @@ def install_compiled_deps_if_possible():
# problem with inline declarations in older libc headers. Setting -fgnu89-inline is a
# workaround.
distro_version = ''.join(exec_cmd(["lsb_release", "-irs"]).lower().split())
- print distro_version
+ print(distro_version)
if distro_version.startswith("centos5."):
env["CFLAGS"] = "-fgnu89-inline"
@@ -381,8 +382,8 @@ if __name__ == "__main__":
if options.print_ld_library_path:
kudu_client_dir = find_kudu_client_install_dir()
- print os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
- os.path.join(kudu_client_dir, 'lib64')])
+ print(os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
+ os.path.join(kudu_client_dir, 'lib64')]))
sys.exit()
logging.basicConfig(level=getattr(logging, options.log_level))
diff --git a/infra/python/deps/find_py26.py b/infra/python/deps/find_py26.py
index 9135469..dc628c7 100755
--- a/infra/python/deps/find_py26.py
+++ b/infra/python/deps/find_py26.py
@@ -21,6 +21,7 @@
# system command to stdout. The script exits with a nonzero exit code if
# Python 2.6+ is not present.
+from __future__ import print_function
import os
import subprocess
import sys
@@ -42,4 +43,5 @@ def detect_python_cmd():
return cmd_path
raise Exception("Could not find minimum required python version 2.6")
-print detect_python_cmd()
+
+print(detect_python_cmd())
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index 3e593c4..354bff4 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -21,6 +21,7 @@
# over which archive type is downloaded and what post-download steps are executed.
# This script requires Python 2.6+.
+from __future__ import print_function
import hashlib
import multiprocessing.pool
import os
@@ -29,7 +30,13 @@ import re
import sys
from random import randint
from time import sleep
-from urllib import urlopen, FancyURLopener
+# The path to import is different for libraries in Python 2 & 3 - this try/catch ensures
+# that this code runs in both
+try:
+ # This should be removed when support for Python2 is dropped
+ from urllib import urlopen, FancyURLopener
+except ImportError:
+ from urllib.request import urlopen, FancyURLopener
NUM_DOWNLOAD_ATTEMPTS = 8
@@ -48,7 +55,7 @@ def check_digest(filename, algorithm, expected_digest):
# Fallback to hardcoded set if hashlib.algorithms_available doesn't exist.
supported_algorithms = set(['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'])
if algorithm not in supported_algorithms:
- print 'Hash algorithm {0} is not supported by hashlib'.format(algorithm)
+ print('Hash algorithm {0} is not supported by hashlib'.format(algorithm))
return False
h = hashlib.new(algorithm)
h.update(open(filename).read())
@@ -60,18 +67,18 @@ def retry(func):
'''Retry decorator.'''
def wrapper(*args, **kwargs):
- for try_num in xrange(NUM_DOWNLOAD_ATTEMPTS):
+ for try_num in range(NUM_DOWNLOAD_ATTEMPTS):
if try_num > 0:
sleep_len = randint(5, 10 * 2 ** try_num)
- print 'Sleeping for {0} seconds before retrying'.format(sleep_len)
+ print('Sleeping for {0} seconds before retrying'.format(sleep_len))
sleep(sleep_len)
try:
result = func(*args, **kwargs)
if result:
return result
except Exception as e:
- print e
- print 'Download failed after several attempts.'
+ print(e)
+ print('Download failed after several attempts.')
sys.exit(1)
return wrapper
@@ -83,7 +90,7 @@ def get_package_info(pkg_name, pkg_version):
# same result is always returned even if the ordering changed on the server.
candidates = []
url = '{0}/simple/{1}/'.format(PYPI_MIRROR, pkg_name)
- print 'Getting package info from {0}'.format(url)
+ print('Getting package info from {0}'.format(url))
# The web page should be in PEP 503 format (https://www.python.org/dev/peps/pep-0503/).
# We parse the page with regex instead of an html parser because that requires
# downloading an extra package before running this script. Since the HTML is guaranteed
@@ -101,7 +108,7 @@ def get_package_info(pkg_name, pkg_version):
file_name.endswith('-{0}.zip'.format(pkg_version))):
candidates.append((file_name, path, hash_algorithm, digest))
if not candidates:
- print 'Could not find archive to download for {0} {1}'.format(pkg_name, pkg_version)
+ print('Could not find archive to download for {0} {1}'.format(pkg_name, pkg_version))
return (None, None, None, None)
return sorted(candidates)[0]
@@ -113,16 +120,16 @@ def download_package(pkg_name, pkg_version):
return False
if os.path.isfile(file_name) and check_digest(file_name, hash_algorithm,
expected_digest):
- print 'File with matching digest already exists, skipping {0}'.format(file_name)
+ print('File with matching digest already exists, skipping {0}'.format(file_name))
return True
downloader = FancyURLopener()
pkg_url = '{0}/packages/{1}'.format(PYPI_MIRROR, path)
- print 'Downloading {0} from {1}'.format(file_name, pkg_url)
+ print('Downloading {0} from {1}'.format(file_name, pkg_url))
downloader.retrieve(pkg_url, file_name)
if check_digest(file_name, hash_algorithm, expected_digest):
return True
else:
- print 'Hash digest check failed in file {0}.'.format(file_name)
+ print('Hash digest check failed in file {0}.'.format(file_name))
return False
def main():