You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2023/02/28 17:12:51 UTC
[impala] 04/06: IMPALA-11952 (part 2): Fix print function syntax
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2b550634d2dbb38d9fda2536e8c3b092787e7456
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sun Feb 26 13:54:52 2023 -0800
IMPALA-11952 (part 2): Fix print function syntax
Python 3 now treats print as a function and requires
the parenthesis in invocation.
print "Hello World!"
is now:
print("Hello World!")
This fixes all locations to use the function
invocation. This is more complicated when the output
is being redirected to a file or when avoiding the
usual newline.
print >> sys.stderr , "Hello World!"
is now:
print("Hello World!", file=sys.stderr)
To support this properly and guarantee equivalent behavior
between python 2 and python 3, all files that use print
now add this import:
from __future__ import print_function
This also fixes random flake8 issues that intersect with
the changes.
Testing:
- check-python-syntax.sh shows no errors related to print
Change-Id: Ib634958369ad777a41e72d80c8053b74384ac351
Reviewed-on: http://gerrit.cloudera.org:8080/19552
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Michael Smith <mi...@cloudera.com>
---
be/src/codegen/gen_ir_descriptions.py | 5 +-
bin/check-rat-report.py | 3 +-
bin/collect_minidumps.py | 11 ++--
bin/compare_branches.py | 33 +++++-----
bin/diagnostics/experimental/plan-graph.py | 1 +
.../experimental/tpcds_run_comparator.py | 1 +
bin/gen_build_version.py | 9 +--
bin/generate_xml_config.py | 13 ++--
bin/get_code_size.py | 1 +
bin/inline_pom.py | 1 +
bin/jenkins/critique-gerrit-review.py | 3 +-
bin/jenkins/dockerized-impala-preserve-vars.py | 1 +
bin/jenkins/populate_m2_directory.py | 1 +
bin/load-data.py | 3 +-
bin/parse-thrift-profile.py | 5 +-
bin/push_to_asf.py | 73 +++++++++++-----------
bin/run-workload.py | 11 ++--
bin/single_node_perf_run.py | 3 +-
bin/validate-unified-backend-test-filters.py | 1 +
common/thrift/generate_error_codes.py | 1 +
common/thrift/generate_metrics.py | 5 +-
infra/deploy/deploy.py | 60 +++++++++---------
lib/python/impala_py_lib/gdb/impala-gdb.py | 1 +
.../impala_py_lib/jenkins/generate_junitxml.py | 1 +
lib/python/impala_py_lib/profiles.py | 3 +-
testdata/bin/generate-schema-statements.py | 40 ++++++------
testdata/bin/generate-test-vectors.py | 17 ++---
testdata/bin/load-tpc-kudu.py | 3 +-
testdata/bin/rewrite-iceberg-metadata.py | 1 +
testdata/bin/wait-for-hiveserver2.py | 13 ++--
testdata/bin/wait-for-metastore.py | 11 ++--
.../common/etc/hadoop/conf/yarn-site.xml.py | 3 +-
testdata/common/cgroups.py | 3 +-
testdata/common/text_delims_table.py | 1 +
testdata/common/widetable.py | 3 +-
tests/benchmark/report_benchmark_results.py | 5 +-
tests/common/impala_cluster.py | 3 +-
tests/common/impala_test_suite.py | 3 +-
tests/common/resource_pool_config.py | 3 +-
tests/comparison/cluster.py | 1 +
tests/comparison/data_generator_mapper.py | 1 +
tests/comparison/db_connection.py | 1 +
tests/comparison/discrepancy_searcher.py | 1 +
tests/comparison/query_generator.py | 1 +
tests/comparison/util/verify-oracle-connection.py | 3 +-
tests/conftest.py | 1 +
tests/custom_cluster/test_blacklist.py | 3 +-
tests/custom_cluster/test_breakpad.py | 3 +-
tests/custom_cluster/test_catalog_hms_failures.py | 3 +-
tests/custom_cluster/test_client_ssl.py | 3 +-
tests/custom_cluster/test_events_custom_configs.py | 1 +
tests/custom_cluster/test_local_catalog.py | 3 +-
tests/custom_cluster/test_query_retries.py | 3 +-
tests/custom_cluster/test_restart_services.py | 3 +-
tests/custom_cluster/test_scratch_disk.py | 3 +-
.../custom_cluster/test_topic_update_frequency.py | 1 +
tests/custom_cluster/test_udf_concurrency.py | 11 ++--
tests/custom_cluster/test_web_pages.py | 1 +
tests/metadata/test_hms_integration.py | 4 +-
tests/query_test/test_aggregation.py | 4 +-
tests/query_test/test_compressed_formats.py | 3 +-
tests/query_test/test_hdfs_caching.py | 3 +-
tests/query_test/test_kudu.py | 3 +-
tests/query_test/test_limit.py | 3 +-
tests/query_test/test_scanners.py | 5 +-
tests/run-tests.py | 11 ++--
tests/shell/test_shell_interactive.py | 5 +-
tests/shell/util.py | 1 +
tests/statestore/test_statestore.py | 3 +-
tests/stress/extract_min_mem.py | 3 +-
tests/stress/runtime_info.py | 1 +
tests/unittests/test_file_parser.py | 7 ++-
tests/util/acid_txn.py | 3 +-
tests/util/run_impyla_http_query.py | 1 +
tests/util/test_file_parser.py | 5 +-
75 files changed, 275 insertions(+), 196 deletions(-)
diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py
index 344491b5c..3ed2935f4 100755
--- a/be/src/codegen/gen_ir_descriptions.py
+++ b/be/src/codegen/gen_ir_descriptions.py
@@ -19,6 +19,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
from string import Template
import os
import shutil
@@ -338,7 +339,7 @@ def move_if_different(src_file, dest_file):
if not os.path.isfile(dest_file) or not filecmp.cmp(src_file, dest_file):
shutil.move(src_file, dest_file)
else:
- print 'Retaining existing file: %s' % (dest_file)
+ print('Retaining existing file: %s' % (dest_file))
BE_PATH = os.path.join(os.environ['IMPALA_HOME'], 'be/generated-sources/impala-ir/')
IR_FUNCTIONS_FILE = 'impala-ir-functions.h'
@@ -352,7 +353,7 @@ if not os.path.exists(BE_PATH):
os.makedirs(BE_PATH)
if __name__ == "__main__":
- print "Generating IR description files"
+ print("Generating IR description files")
enums_file = open(TMP_IR_FUNCTIONS_PATH, 'w')
enums_file.write(enums_preamble)
diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py
index b296c6dec..3c6870bee 100755
--- a/bin/check-rat-report.py
+++ b/bin/check-rat-report.py
@@ -33,6 +33,7 @@
# time, and the RAT JAR is not included in the Impala repo; it must be downloaded
# separately.
+from __future__ import print_function
import fnmatch
import re
import sys
@@ -74,5 +75,5 @@ for r in resources:
if not all_ok:
sys.exit(1)
-print 'OK'
+print('OK')
sys.exit(0)
diff --git a/bin/collect_minidumps.py b/bin/collect_minidumps.py
index 30d6d1f80..db233a946 100755
--- a/bin/collect_minidumps.py
+++ b/bin/collect_minidumps.py
@@ -24,6 +24,7 @@
# --role_name=statestored --max_output_size=50000000 --end_time=1463033495000 \
# --output_file_path=/tmp/minidump_package.tar.gz
+from __future__ import print_function
import os
import re
import sys
@@ -162,7 +163,7 @@ def get_config_parameter_value(conf_dir, role_name, config_parameter_name):
if m:
config_parameter_value = m.group(1)
except IOError as e:
- print >> sys.stderr, 'Error: Unable to open "{0}".'.format(file_path)
+ print('Error: Unable to open "{0}".'.format(file_path), file=sys.stderr)
sys.exit(1)
return config_parameter_value
@@ -179,7 +180,7 @@ def get_minidump_dir(conf_dir, role_name):
result = os.path.join(minidump_path, role_name)
if not os.path.isdir(result):
msg = 'Error: minidump directory does not exist.'
- print >> sys.stderr, msg
+ print(msg, file=sys.stderr)
sys.exit(1)
return result
@@ -202,11 +203,11 @@ def main():
options, args = parser.parse_args()
if not options.conf_dir:
msg = 'Error: conf_dir is not specified.'
- print >> sys.stderr, msg
+ print(msg, file=sys.stderr)
sys.exit(1)
if not options.output_file_path:
msg = 'Error: output_file_path is not specified.'
- print >> sys.stderr, msg
+ print(msg, file=sys.stderr)
sys.exit(1)
minidump_dir = get_minidump_dir(options.conf_dir, options.role_name)
@@ -216,7 +217,7 @@ def main():
end_time=options.end_time,
output_file_path=options.output_file_path)
status, msg = file_archiver.make_tarball()
- print >> sys.stderr, msg
+ print(msg, file=sys.stderr)
sys.exit(status)
if __name__ == '__main__':
diff --git a/bin/compare_branches.py b/bin/compare_branches.py
index da1d5f8b9..143fa7242 100755
--- a/bin/compare_branches.py
+++ b/bin/compare_branches.py
@@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Future imports must happen at the beginning of the file
+from __future__ import print_function
+
HELP = '''
Compares two specified branches, using the Gerrit Change-Id as the
primary identifier. Ignored commits can be added via a JSON
@@ -171,7 +174,7 @@ def cherrypick(cherry_pick_hashes, full_target_branch_name, partial_ok):
Note that this function does not push to the remote.
"""
- print "Cherrypicking %d changes." % (len(cherry_pick_hashes),)
+ print("Cherrypicking %d changes." % (len(cherry_pick_hashes),))
if len(cherry_pick_hashes) == 0:
return
@@ -181,8 +184,8 @@ def cherrypick(cherry_pick_hashes, full_target_branch_name, partial_ok):
target_branch_sha = subprocess.check_output(
['git', 'rev-parse', full_target_branch_name]).strip()
if head_sha != target_branch_sha:
- print "Cannot cherrypick because %s (%s) and HEAD (%s) are divergent." % (
- full_target_branch_name, target_branch_sha, head_sha)
+ print("Cannot cherrypick because %s (%s) and HEAD (%s) are divergent." % (
+ full_target_branch_name, target_branch_sha, head_sha))
sys.exit(1)
cherry_pick_hashes.reverse()
@@ -192,7 +195,7 @@ def cherrypick(cherry_pick_hashes, full_target_branch_name, partial_ok):
if ret != 0:
if partial_ok and i > 0:
subprocess.check_call(['git', 'cherry-pick', '--abort'])
- print "Failed to cherry-pick %s; stopping picks." % (cherry_pick_hash,)
+ print("Failed to cherry-pick %s; stopping picks." % (cherry_pick_hash,))
return
else:
raise Exception("Failed to cherry-pick: %s" % (cherry_pick_hash,))
@@ -238,10 +241,10 @@ def main():
commits_ignored = [] # Track commits actually ignored for debug logging
cherry_pick_hashes = []
- print '-' * 80
- print 'Commits in {0} but not in {1}:'.format(
- full_source_branch_name, full_target_branch_name)
- print '-' * 80
+ print('-' * 80)
+ print('Commits in {0} but not in {1}:'.format(
+ full_source_branch_name, full_target_branch_name))
+ print('-' * 80)
jira_keys = []
jira_key_pat = re.compile(r'(IMPALA-\d+)')
skip_commits_matching = options.skip_commits_matching.format(
@@ -267,17 +270,17 @@ def main():
logging.debug("NOT ignoring commit {0} since not in ignored commits ({1},{2})"
.format(commit_hash, options.source_branch, options.target_branch))
if not change_in_target and not ignore_by_config and not ignore_by_commit_message:
- print u'{0} {1} ({2}) - {3}'\
- .format(commit_hash, msg.decode('utf8'), date, author.decode('utf8'))\
- .encode('utf8')
+ print(u'{0} {1} ({2}) - {3}'
+ .format(commit_hash, msg.decode('utf8'), date, author.decode('utf8'))
+ .encode('utf8'))
cherry_pick_hashes.append(commit_hash)
jira_keys += jira_key_pat.findall(msg)
- print '-' * 80
+ print('-' * 80)
- print "Jira keys referenced (Note: not all commit messages will reference a jira key):"
- print ','.join(jira_keys)
- print '-' * 80
+ print("Jira keys referenced (Note: not all commit messages will reference a jira key):")
+ print(','.join(jira_keys))
+ print('-' * 80)
logging.debug("Commits actually ignored (change was not in target): {0}"
.format(pformat(commits_ignored)))
diff --git a/bin/diagnostics/experimental/plan-graph.py b/bin/diagnostics/experimental/plan-graph.py
index 566d195df..03e06d663 100755
--- a/bin/diagnostics/experimental/plan-graph.py
+++ b/bin/diagnostics/experimental/plan-graph.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License
+from __future__ import print_function
import argparse
import math
import re
diff --git a/bin/diagnostics/experimental/tpcds_run_comparator.py b/bin/diagnostics/experimental/tpcds_run_comparator.py
index 3c635cd6b..80b787f0f 100755
--- a/bin/diagnostics/experimental/tpcds_run_comparator.py
+++ b/bin/diagnostics/experimental/tpcds_run_comparator.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License
+from __future__ import print_function
import argparse
import csv
import math
diff --git a/bin/gen_build_version.py b/bin/gen_build_version.py
index 4030d61ca..c6468fa69 100755
--- a/bin/gen_build_version.py
+++ b/bin/gen_build_version.py
@@ -22,6 +22,7 @@
# This script generates be/src/common/version.h which contains the build version based
# on the git hash.
+from __future__ import print_function
import os
from subprocess import call
@@ -46,15 +47,15 @@ version_file_exists = os.path.isfile(VERSION_FILE_NAME)
# If we have a version file and cannot obtain a git hash, skip generating a new
# version file.
if version_file_exists and not can_obtain_git_hash:
- print "Cannot obtain git hash, using existing version file."
+ print("Cannot obtain git hash, using existing version file.")
else:
# Remove existing version files only if they exist.
# TODO: Might be useful to make a common utility function remove_if_clean.
if version_file_exists:
- print 'Removing existing file: %s' % (VERSION_FILE_NAME)
+ print('Removing existing file: %s' % (VERSION_FILE_NAME))
os.remove(VERSION_FILE_NAME)
if os.path.isfile(VERSION_CC_FILE_NAME):
- print 'Removing existing file: %s' % (VERSION_CC_FILE_NAME)
+ print('Removing existing file: %s' % (VERSION_CC_FILE_NAME))
os.remove(VERSION_CC_FILE_NAME)
# SAVE_VERSION_SCRIPT will generate a dummy version.info file if we cannot obtain the
@@ -74,7 +75,7 @@ try:
finally:
version_file.close()
-print '\n'.join([version, git_hash, build_time])
+print('\n'.join([version, git_hash, build_time]))
file_contents = """
//
diff --git a/bin/generate_xml_config.py b/bin/generate_xml_config.py
index 915e3e276..e8f03dd11 100755
--- a/bin/generate_xml_config.py
+++ b/bin/generate_xml_config.py
@@ -40,6 +40,7 @@ REPL:
"""
from __future__ import with_statement
+from __future__ import print_function
import imp
import os
import re
@@ -77,7 +78,7 @@ def dump_config(d, source_path, out):
-->
<configuration>""".format(source_path=os.path.abspath(source_path))
- print >>out, dedent(header)
+ print(dedent(header), file=out)
for k, v in sorted(d.iteritems()):
try:
k_new = _substitute_env_vars(k)
@@ -87,24 +88,24 @@ def dump_config(d, source_path, out):
except KeyError as e:
raise Exception("failed environment variable substitution for value {k}: {e}"
.format(k=k, e=e))
- print >>out, """\
+ print("""\
<property>
<name>{name}</name>
<value>{value}</value>
- </property>""".format(name=xmlescape(k_new), value=xmlescape(v_new))
- print >>out, "</configuration>"
+ </property>""".format(name=xmlescape(k_new), value=xmlescape(v_new)), file=out)
+ print("</configuration>", file=out)
def main():
if len(sys.argv) != 3:
- print >>sys.stderr, "usage: {prog} <template> <out>".format(prog=sys.argv[0])
+ print("usage: {prog} <template> <out>".format(prog=sys.argv[0]), file=sys.stderr)
sys.exit(1)
_, in_path, out_path = sys.argv
try:
mod = imp.load_source('template', in_path)
except: # noqa
- print >>sys.stderr, "Unable to load template: %s" % in_path
+ print("Unable to load template: %s" % in_path, file=sys.stderr)
raise
conf = mod.__dict__.get('CONFIG')
if not isinstance(conf, dict):
diff --git a/bin/get_code_size.py b/bin/get_code_size.py
index 9a2e78298..b7d6eda6f 100755
--- a/bin/get_code_size.py
+++ b/bin/get_code_size.py
@@ -19,6 +19,7 @@
# This tool walks the build directory (release by default) and will print the text, data,
# and bss section sizes of the archives.
+from __future__ import print_function
import fnmatch
import os
import re
diff --git a/bin/inline_pom.py b/bin/inline_pom.py
index a80574363..1b021683a 100755
--- a/bin/inline_pom.py
+++ b/bin/inline_pom.py
@@ -21,6 +21,7 @@
#
# Usage: inline_pom.py <pom.xml>...
+from __future__ import print_function
import re
import sys
from tempfile import mkstemp
diff --git a/bin/jenkins/critique-gerrit-review.py b/bin/jenkins/critique-gerrit-review.py
index 466a1ad18..7f44f47bd 100755
--- a/bin/jenkins/critique-gerrit-review.py
+++ b/bin/jenkins/critique-gerrit-review.py
@@ -36,6 +36,7 @@
# TODO: generalise to other warnings
# * clang-tidy
+from __future__ import print_function
from argparse import ArgumentParser
from collections import defaultdict
import json
@@ -229,6 +230,6 @@ if __name__ == "__main__":
comments = get_flake8_comments(revision)
merge_comments(comments, get_misc_comments(revision))
review_input = {"comments": comments}
- print json.dumps(review_input, indent=True)
+ print(json.dumps(review_input, indent=True))
if not args.dryrun:
post_review_to_gerrit(review_input)
diff --git a/bin/jenkins/dockerized-impala-preserve-vars.py b/bin/jenkins/dockerized-impala-preserve-vars.py
index f36a45dab..24abea465 100755
--- a/bin/jenkins/dockerized-impala-preserve-vars.py
+++ b/bin/jenkins/dockerized-impala-preserve-vars.py
@@ -28,6 +28,7 @@
# If an environment variable is not defined in the current environment,
# it is omitted with a warning.
+from __future__ import print_function
import sys
import os
diff --git a/bin/jenkins/populate_m2_directory.py b/bin/jenkins/populate_m2_directory.py
index 15701898a..de56808df 100755
--- a/bin/jenkins/populate_m2_directory.py
+++ b/bin/jenkins/populate_m2_directory.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import json
import subprocess
import os
diff --git a/bin/load-data.py b/bin/load-data.py
index ef407306e..362ec8ee3 100755
--- a/bin/load-data.py
+++ b/bin/load-data.py
@@ -20,6 +20,7 @@
# This script is used to load the proper datasets for the specified workloads. It loads
# all data via Hive except for parquet data which needs to be loaded via Impala.
# Most ddl commands are executed by Impala.
+from __future__ import print_function
import collections
import getpass
import logging
@@ -102,7 +103,7 @@ HIVE_CMD = os.path.join(os.environ['HIVE_HOME'], 'bin/beeline')
hive_auth = "auth=none"
if options.use_kerberos:
if not options.principal:
- print "--principal is required when --use_kerberos is specified"
+ print("--principal is required when --use_kerberos is specified")
exit(1)
hive_auth = "principal=" + options.principal
diff --git a/bin/parse-thrift-profile.py b/bin/parse-thrift-profile.py
index 55cf1f2b6..7661e1ff0 100755
--- a/bin/parse-thrift-profile.py
+++ b/bin/parse-thrift-profile.py
@@ -39,6 +39,7 @@
# 2018-04-13T15:06:34.144000 e44af7f93edb8cd6:1b1f801600000000 TRuntimeProfileTree(nodes=[TRuntimeProf...
+from __future__ import print_function
from impala_py_lib import profiles
import sys
@@ -47,9 +48,9 @@ if len(sys.argv) == 1 or sys.argv[1] == "-":
elif len(sys.argv) == 2:
input_data = file(sys.argv[1])
else:
- print >> sys.stderr, "Usage: %s [file]" % (sys.argv[0],)
+ print("Usage: %s [file]" % (sys.argv[0],), file=sys.stderr)
sys.exit(1)
for line in input_data:
tree = profiles.decode_profile_line(line)
- print tree
+ print(tree)
diff --git a/bin/push_to_asf.py b/bin/push_to_asf.py
index fab6cfcc6..fac612c4a 100755
--- a/bin/push_to_asf.py
+++ b/bin/push_to_asf.py
@@ -35,6 +35,7 @@
# TODO: Improve console output: replace 'print' with format strings
# and use sys.stderr/sys.stdout.
+from __future__ import print_function
import logging
import optparse
import os
@@ -87,10 +88,10 @@ def confirm_prompt(prompt):
if the user confirms.
"""
while True:
- print prompt, "[Y/n]:",
+ print(prompt, "[Y/n]:", end=' ')
if not os.isatty(sys.stdout.fileno()):
- print "Not running interactively. Assuming 'N'."
+ print("Not running interactively. Assuming 'N'.")
return False
r = raw_input().strip().lower()
@@ -115,14 +116,14 @@ def check_apache_remote():
['git', 'config', '--local', '--get',
'remote.' + OPTIONS.apache_remote + '.url']).strip()
except subprocess.CalledProcessError:
- print >>sys.stderr, "No remote named " + OPTIONS.apache_remote + \
- ". Please set one up, for example with: "
- print >>sys.stderr, " git remote add apache", APACHE_REPO
+ print("No remote named " + OPTIONS.apache_remote
+ + ". Please set one up, for example with: ", file=sys.stderr)
+ print(" git remote add apache", APACHE_REPO, file=sys.stderr)
sys.exit(1)
if url != APACHE_REPO:
- print >>sys.stderr, "Unexpected URL for remote " + OPTIONS.apache_remote + "."
- print >>sys.stderr, " Got: ", url
- print >>sys.stderr, " Expected:", APACHE_REPO
+ print("Unexpected URL for remote " + OPTIONS.apache_remote + ".", file=sys.stderr)
+ print(" Got: ", url, file=sys.stderr)
+ print(" Expected:", APACHE_REPO, file=sys.stderr)
sys.exit(1)
@@ -135,15 +136,15 @@ def check_gerrit_remote():
url = check_output(['git', 'config', '--local', '--get',
'remote.' + OPTIONS.gerrit_remote + '.url']).strip()
except subprocess.CalledProcessError:
- print >>sys.stderr, "No remote named " + OPTIONS.gerrit_remote + \
- ". Please set one up following "
- print >>sys.stderr, "the contributor guide."
+ print("No remote named " + OPTIONS.gerrit_remote
+ + ". Please set one up following ", file=sys.stderr)
+ print("the contributor guide.", file=sys.stderr)
sys.exit(1)
if not GERRIT_URL_RE.match(url):
- print >>sys.stderr, "Unexpected URL for remote " + OPTIONS.gerrit_remote
- print >>sys.stderr, " Got: ", url
- print >>sys.stderr, " Expected to find host '%s' in the URL" % GERRIT_HOST
+ print("Unexpected URL for remote " + OPTIONS.gerrit_remote, file=sys.stderr)
+ print(" Got: ", url, file=sys.stderr)
+ print(" Expected URL to match '%s'" % GERRIT_URL_RE, file=sys.stderr)
sys.exit(1)
@@ -211,40 +212,40 @@ def do_update(branch, gerrit_sha, apache_sha):
# must have gotten committed to Apache outside of gerrit, and we'd need some
# manual intervention.
if not is_fast_forward(apache_sha, gerrit_sha):
- print >>sys.stderr, "Cannot update branch '%s' from gerrit:" % branch
- print >>sys.stderr, "Apache revision %s is not an ancestor of gerrit revision %s" % (
- apache_sha[:8], gerrit_sha[:8])
- print >>sys.stderr,\
- "Something must have been committed to Apache and bypassed gerrit."
- print >>sys.stderr, "Manual intervention is required."
+ print("Cannot update branch '%s' from gerrit:" % branch, file=sys.stderr)
+ print("Apache revision %s is not an ancestor of gerrit revision %s" % (
+ apache_sha[:8], gerrit_sha[:8]), file=sys.stderr)
+ print("Something must have been committed to Apache and bypassed gerrit.",
+ file=sys.stderr)
+ print("Manual intervention is required.", file=sys.stderr)
sys.exit(1)
# List the commits that are going to be pushed to the ASF, so that the committer
# can verify and "sign off".
commits = rev_list("%s..%s" % (apache_sha, gerrit_sha))
commits.reverse() # Display from oldest to newest.
- print "-" * 60
- print Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" %\
- len(commits)) + Colors.RESET
+ print("-" * 60)
+ print(Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" %
+ len(commits)) + Colors.RESET)
push_sha = None
for sha in commits:
oneline = describe_commit(sha)
- print " ", oneline
+ print(" ", oneline)
committer = get_committer_email(sha)
if committer != get_my_email():
- print Colors.RED + " !!! Committed by someone else (%s) !!!" %\
- committer, Colors.RESET
+ print(Colors.RED + " !!! Committed by someone else (%s) !!!" %
+ committer, Colors.RESET)
if not confirm_prompt(Colors.RED +\
" !!! Are you sure you want to push on behalf of another committer?" +\
Colors.RESET):
# Even if they don't want to push this commit, we could still push any
# earlier commits that the user _did_ author.
if push_sha is not None:
- print "... will still update to prior commit %s..." % push_sha
+ print("... will still update to prior commit %s..." % push_sha)
break
push_sha = sha
if push_sha is None:
- print "Nothing to push"
+ print("Nothing to push")
return
# Everything has been confirmed. Do the actual push
@@ -252,11 +253,11 @@ def do_update(branch, gerrit_sha, apache_sha):
if OPTIONS.dry_run:
cmd.append('--dry-run')
cmd.append('%s:refs/heads/%s' % (push_sha, branch))
- print Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd)
+ print(Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd))
subprocess.check_call(cmd)
- print Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) +\
- Colors.RESET
- print
+ print(Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha)
+ + Colors.RESET)
+ print()
def main():
@@ -295,14 +296,14 @@ def main():
apache_branches = get_branches(OPTIONS.apache_remote)
for branch, apache_sha in sorted(apache_branches.iteritems()):
gerrit_sha = rev_parse("remotes/" + OPTIONS.gerrit_remote + "/" + branch)
- print "Branch '%s':\t" % branch,
+ print("Branch '%s':\t" % branch, end='')
if gerrit_sha is None:
- print Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET
+ print(Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET)
continue
if gerrit_sha == apache_sha:
- print Colors.GREEN, "up to date", Colors.RESET
+ print(Colors.GREEN, "up to date", Colors.RESET)
continue
- print Colors.YELLOW, "needs update", Colors.RESET
+ print(Colors.YELLOW, "needs update", Colors.RESET)
do_update(branch, gerrit_sha, apache_sha)
diff --git a/bin/run-workload.py b/bin/run-workload.py
index 8d2d2f75b..abe95ceb1 100755
--- a/bin/run-workload.py
+++ b/bin/run-workload.py
@@ -27,6 +27,7 @@
# - Stores the execution details in JSON format.
#
+from __future__ import print_function
import getpass
import json
import logging
@@ -158,16 +159,16 @@ def prettytable_print(results, failed=False):
table.float_format = '.2'
# Group the results by table format.
for table_format_str, gr in groupby(results, lambda x: x.query.table_format_str):
- print "Table Format: %s" % table_format_str
+ print("Table Format: %s" % table_format_str)
for result in gr:
start_time = result.start_time.strftime("%Y-%m-%d %H:%M:%S") if result.start_time \
is not None else '-'
row = [result.query.name, start_time, result.time_taken, result.client_name]
if failed: row.append(result.query_error)
table.add_row(row)
- print table.get_string(sortby='Client ID')
+ print(table.get_string(sortby='Client ID'))
table.clear_rows()
- print str()
+ print(str())
def print_result_summary(results):
"""Print failed and successfull queries for a given result list"""
@@ -271,8 +272,8 @@ if __name__ == "__main__":
if not all(result.success for result in workload_runner.results): exit_code = 1
# Print the results
- print "\nWorkload: {0}, Scale Factor: {1}\n".format(
- workload_runner.workload.name.upper(), workload_runner.scale_factor)
+ print("\nWorkload: {0}, Scale Factor: {1}\n".format(
+ workload_runner.workload.name.upper(), workload_runner.scale_factor))
print_result_summary(workload_runner.results)
# Store the results
diff --git a/bin/single_node_perf_run.py b/bin/single_node_perf_run.py
index e9cd33d80..e1b74de64 100755
--- a/bin/single_node_perf_run.py
+++ b/bin/single_node_perf_run.py
@@ -69,6 +69,7 @@
# --start_minicluster start a new Hadoop minicluster
# --ninja use ninja, rather than Make, as the build tool
+from __future__ import print_function
from optparse import OptionParser
from tempfile import mkdtemp
@@ -208,7 +209,7 @@ def backup_workloads():
temp_dir = mkdtemp()
sh.cp(os.path.join(IMPALA_HOME, "testdata", "workloads"),
temp_dir, R=True, _out=sys.stdout, _err=sys.stderr)
- print "Backed up workloads to {0}".format(temp_dir)
+ print("Backed up workloads to {0}".format(temp_dir))
return temp_dir
diff --git a/bin/validate-unified-backend-test-filters.py b/bin/validate-unified-backend-test-filters.py
index 6f8602da4..a978917b2 100755
--- a/bin/validate-unified-backend-test-filters.py
+++ b/bin/validate-unified-backend-test-filters.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import os
import subprocess
import sys
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index 3bbe83672..02b7029ef 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -25,6 +25,7 @@
#
# TODO Add support for SQL Error Codes
# https://msdn.microsoft.com/en-us/library/ms714687%28v=vs.85%29.aspx
+from __future__ import print_function
error_codes = (
("OK", 0, ""),
diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py
index 42b291e0a..8534d0e0e 100755
--- a/common/thrift/generate_metrics.py
+++ b/common/thrift/generate_metrics.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import sys
import os
import re
@@ -129,13 +130,13 @@ def metric_to_mdl(m):
"""Returns the metric in the mdl format, or None if the metric isn't supported."""
# TODO: Stamp out metrics with arguments, e.g. output each rpc call_duration metric.
if '$0' in m['key']:
- print >>sys.stderr, "Skipping metrics with unbound argument, key=%s" % m['key']
+ print("Skipping metrics with unbound argument, key=%s" % m['key'], file=sys.stderr)
return None
# TODO: Stamp out individual metrics for other metric types.
SUPPORTED_METRIC_KINDS = ['COUNTER', 'GAUGE']
if m['kind'] not in SUPPORTED_METRIC_KINDS:
- print >>sys.stderr, "Skipping %s metric %s" % (m['kind'], m['key'])
+ print("Skipping %s metric %s" % (m['kind'], m['key']), file=sys.stderr)
return None
return dict(
diff --git a/infra/deploy/deploy.py b/infra/deploy/deploy.py
index 1665e5182..58ebb69fa 100644
--- a/infra/deploy/deploy.py
+++ b/infra/deploy/deploy.py
@@ -32,6 +32,7 @@
# Creates a new Impala_Kudu service called "new_service" using /data/impala/
# for its scratch directories.
+from __future__ import print_function
import argparse
import hashlib
import os
@@ -134,11 +135,11 @@ def find_dependencies(args, cluster):
if not found:
raise Exception("Could not find dependency service (type %s, name %s)" %
(service_type, arg))
- print "Found explicit dependency service %s" % (found.name)
+ print("Found explicit dependency service %s" % (found.name))
deps.append(found)
else:
if not required:
- print "Skipping optional dependency of type %s" % (service_type,)
+ print("Skipping optional dependency of type %s" % (service_type,))
continue
if len(candidates) > 1:
raise Exception("Found %d possible implicit dependency services of type %s" %
@@ -148,7 +149,7 @@ def find_dependencies(args, cluster):
(service_type,))
else:
found = candidates.values()[0]
- print "Found implicit dependency service %s" % (found.name,)
+ print("Found implicit dependency service %s" % (found.name,))
deps.append(found)
return deps
@@ -157,7 +158,7 @@ def check_new_service_does_not_exist(api, cluster, new_name):
if service.displayName == new_name:
raise Exception("New service name %s already in use" % (new_name,))
- print "New service name %s is not in use" % (new_name,)
+ print("New service name %s is not in use" % (new_name,))
def find_template_service(api, cluster, based_on):
template_service = None
@@ -166,7 +167,7 @@ def find_template_service(api, cluster, based_on):
if service.type != "IMPALA":
raise Exception("Based-on service %s is of wrong type %s" %
(based_on, service.type))
- print "Found based-on service: %s" % (based_on,)
+ print("Found based-on service: %s" % (based_on,))
template_service = service
if based_on and not template_service:
@@ -177,10 +178,10 @@ def find_template_service(api, cluster, based_on):
def find_master_host(api, cm_hostname, master_hostname):
for h in api.get_all_hosts():
if master_hostname and h.hostname == master_hostname:
- print "Found master host %s" % (master_hostname,)
+ print("Found master host %s" % (master_hostname,))
return h
elif not master_hostname and h.hostname == cm_hostname:
- print "Found implicit master host on CM host %s" % (cm_hostname,)
+ print("Found implicit master host on CM host %s" % (cm_hostname,))
return h
if master_hostname:
@@ -225,11 +226,11 @@ def get_best_parcel(api, cluster):
parcel = None
if parcel:
- print "Chose best parcel %s-%s (stage %s)" % (parcel.product,
+ print("Chose best parcel %s-%s (stage %s)" % (parcel.product,
parcel.version,
- parcel.stage)
+ parcel.stage))
else:
- print "Found no candidate parcels"
+ print("Found no candidate parcels")
return parcel
@@ -238,9 +239,9 @@ def ensure_parcel_repo_added(api):
config = cm.get_config(view='summary')
parcel_urls = config.get("REMOTE_PARCEL_REPO_URLS", "").split(",")
if IMPALA_KUDU_PARCEL_URL in parcel_urls:
- print "Impala_Kudu parcel URL already present"
+ print("Impala_Kudu parcel URL already present")
else:
- print "Adding Impala_Kudu parcel URL"
+ print("Adding Impala_Kudu parcel URL")
parcel_urls.append(IMPALA_KUDU_PARCEL_URL)
config["REMOTE_PARCEL_REPO_URLS"] = ",".join(parcel_urls)
cm.update_config(config)
@@ -252,8 +253,8 @@ def wait_for_parcel_stage(cluster, parcel, stage):
return
if new_parcel.state.errors:
raise Exception(str(new_parcel.state.errors))
- print "progress: %s / %s" % (new_parcel.state.progress,
- new_parcel.state.totalProgress)
+ print("progress: %s / %s" % (new_parcel.state.progress,
+ new_parcel.state.totalProgress))
time.sleep(1)
else:
raise Exception("Parcel %s-%s did not reach stage %s in %d seconds" %
@@ -262,33 +263,33 @@ def wait_for_parcel_stage(cluster, parcel, stage):
def ensure_parcel_activated(cluster, parcel):
parcel_stage = parcel.stage
if parcel_stage == "AVAILABLE_REMOTELY":
- print "Downloading parcel: %s-%s " % (parcel.product, parcel.version)
+ print("Downloading parcel: %s-%s " % (parcel.product, parcel.version))
parcel.start_download()
wait_for_parcel_stage(cluster, parcel, "DOWNLOADED")
- print "Downloaded parcel: %s-%s " % (parcel.product, parcel.version)
+ print("Downloaded parcel: %s-%s " % (parcel.product, parcel.version))
parcel_stage = "DOWNLOADED"
if parcel_stage == "DOWNLOADED":
- print "Distributing parcel: %s-%s " % (parcel.product, parcel.version)
+ print("Distributing parcel: %s-%s " % (parcel.product, parcel.version))
parcel.start_distribution()
wait_for_parcel_stage(cluster, parcel, "DISTRIBUTED")
- print "Distributed parcel: %s-%s " % (parcel.product, parcel.version)
+ print("Distributed parcel: %s-%s " % (parcel.product, parcel.version))
parcel_stage = "DISTRIBUTED"
if parcel_stage == "DISTRIBUTED":
- print "Activating parcel: %s-%s " % (parcel.product, parcel.version)
+ print("Activating parcel: %s-%s " % (parcel.product, parcel.version))
parcel.activate()
wait_for_parcel_stage(cluster, parcel, "ACTIVATED")
- print "Activated parcel: %s-%s " % (parcel.product, parcel.version)
+ print("Activated parcel: %s-%s " % (parcel.product, parcel.version))
parcel_stage = "ACTIVATED"
- print "Parcel %s-%s is activated" % (parcel.product, parcel.version)
+ print("Parcel %s-%s is activated" % (parcel.product, parcel.version))
def print_configs(entity_name, config_dict):
for attr, value in config_dict.iteritems():
- print "Set %s config %s=\'%s\'" % (entity_name, attr, value)
+ print("Set %s config %s=\'%s\'" % (entity_name, attr, value))
def create_new_service(api, cluster, new_name, deps, scratch_dirs, master_host):
new_service = cluster.create_service(new_name, "IMPALA")
- print "Created new service %s" % (new_name,)
+ print("Created new service %s" % (new_name,))
service_config = {}
for d in deps:
@@ -312,13 +313,13 @@ def create_new_service(api, cluster, new_name, deps, scratch_dirs, master_host):
md5.update(h.hostId)
new_role_name = "%s-%s-%s" % (new_name, rcg.roleType, md5.hexdigest())
new_service.create_role(new_role_name, rcg.roleType, h.hostId)
- print "Created new role %s" % (new_role_name,)
+ print("Created new role %s" % (new_role_name,))
else:
md5 = hashlib.md5()
md5.update(master_host.hostId)
new_role_name = "%s-%s-%s" % (new_name, rcg.roleType, md5.hexdigest())
new_service.create_role(new_role_name, rcg.roleType, master_host.hostId)
- print "Created new role %s" % (new_role_name,)
+ print("Created new role %s" % (new_role_name,))
def transform_path(rcg_name, rcg_config_dict, rcg_config_name):
# TODO: Do a better job with paths where the role type is embedded.
@@ -379,7 +380,7 @@ def transform_rcg_config(rcg):
def clone_existing_service(cluster, new_name, template_service):
new_service = cluster.create_service(new_name, "IMPALA")
- print "Created new service %s" % (new_name,)
+ print("Created new service %s" % (new_name,))
service_config, _ = template_service.get_config()
service_config["impala_service_env_safety_valve"] = "IMPALA_KUDU=1"
@@ -395,7 +396,7 @@ def clone_existing_service(cluster, new_name, template_service):
new_rcg = new_service.create_role_config_group(new_rcg_name,
new_rcg_name,
old_rcg.roleType)
- print "Created new rcg %s" % (new_rcg_name,)
+ print("Created new rcg %s" % (new_rcg_name,))
else:
new_rcg = new_service.get_role_config_group("%s-%s-BASE" % (new_name,
old_rcg.roleType))
@@ -414,7 +415,7 @@ def clone_existing_service(cluster, new_name, template_service):
new_role = new_service.create_role(new_role_name,
new_rcg.roleType,
old_role.hostRef.hostId)
- print "Created new role %s" % (new_role_name,)
+ print("Created new role %s" % (new_role_name,))
new_role_names.append(new_role.name)
new_rcg.move_roles(new_role_names)
@@ -448,7 +449,8 @@ def main():
parcel = get_best_parcel(api, cluster)
if parcel:
break
- print "Could not find parcel in attempt %d, will sleep and retry" % (attempt,)
+ print("Could not find parcel in attempt %d, will sleep and retry" %
+ (attempt,))
time.sleep(1)
else:
raise Exception("No parcel showed up in %d seconds" % (MAX_PARCEL_REPO_WAIT_SECS,))
diff --git a/lib/python/impala_py_lib/gdb/impala-gdb.py b/lib/python/impala_py_lib/gdb/impala-gdb.py
index 48e2662cf..1bf81eabc 100644
--- a/lib/python/impala_py_lib/gdb/impala-gdb.py
+++ b/lib/python/impala_py_lib/gdb/impala-gdb.py
@@ -19,6 +19,7 @@
# A collection of useful Python GDB modules and commands for
# debugging Impala core dumps.
#
+from __future__ import print_function
import gdb
from collections import defaultdict
diff --git a/lib/python/impala_py_lib/jenkins/generate_junitxml.py b/lib/python/impala_py_lib/jenkins/generate_junitxml.py
index 285aa5d9b..1f6ba6b60 100755
--- a/lib/python/impala_py_lib/jenkins/generate_junitxml.py
+++ b/lib/python/impala_py_lib/jenkins/generate_junitxml.py
@@ -21,6 +21,7 @@ A script for generating arbitrary junit XML reports while building Impala.
These files will be consumed by jenkins.impala.io to generate reports for
easier triaging of build and setup errors.
"""
+from __future__ import print_function
import argparse
import codecs
import errno
diff --git a/lib/python/impala_py_lib/profiles.py b/lib/python/impala_py_lib/profiles.py
index a2f6aebb3..fc7a37298 100644
--- a/lib/python/impala_py_lib/profiles.py
+++ b/lib/python/impala_py_lib/profiles.py
@@ -18,6 +18,7 @@
# This file contains library functions to decode and access Impala query profiles.
+from __future__ import print_function
import base64
import datetime
import zlib
@@ -30,7 +31,7 @@ def decode_profile_line(line):
space_separated = line.split(" ")
if len(space_separated) == 3:
ts = int(space_separated[0])
- print datetime.datetime.fromtimestamp(ts / 1000.0).isoformat(), space_separated[1]
+ print(datetime.datetime.fromtimestamp(ts / 1000.0).isoformat(), space_separated[1])
base64_encoded = space_separated[2]
elif len(space_separated) == 1:
base64_encoded = space_separated[0]
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 8f28e310d..acd369db4 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -94,6 +94,7 @@
# This should be used sparingly, because these commands are executed
# serially.
#
+from __future__ import print_function
import collections
import csv
import glob
@@ -140,7 +141,7 @@ parser.add_option("--hdfs_namenode", dest="hdfs_namenode", default="localhost:20
(options, args) = parser.parse_args()
if options.workload is None:
- print "A workload name must be specified."
+ print("A workload name must be specified.")
parser.print_help()
sys.exit(1)
@@ -604,7 +605,7 @@ def eval_section(section_str):
# features (e.g. "for i in {1..n}")
p = subprocess.Popen(['/bin/bash', '-c', cmd], stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
- if stderr: print stderr
+ if stderr: print(stderr)
assert p.returncode == 0
return stdout.strip()
@@ -638,30 +639,31 @@ def generate_statements(output_name, test_vectors, sections,
table_name = section['BASE_TABLE_NAME'].strip()
if table_names and (table_name.lower() not in table_names):
- print 'Skipping table: %s.%s, table is not in specified table list' % (db, table_name)
+ print('Skipping table: %s.%s, table is not in specified table list' %
+ (db, table_name))
continue
# Check Hive version requirement, if present.
if section['HIVE_MAJOR_VERSION'] and \
section['HIVE_MAJOR_VERSION'].strip() != \
os.environ['IMPALA_HIVE_MAJOR_VERSION'].strip():
- print "Skipping table '{0}.{1}': wrong Hive major version".format(db, table_name)
+ print("Skipping table '{0}.{1}': wrong Hive major version".format(db, table_name))
continue
if table_format in schema_only_constraints and \
table_name.lower() not in schema_only_constraints[table_format]:
- print ('Skipping table: %s.%s, \'only\' constraint for format did not '
- 'include this table.') % (db, table_name)
+ print(('Skipping table: %s.%s, \'only\' constraint for format did not '
+ 'include this table.') % (db, table_name))
continue
if schema_include_constraints[table_name.lower()] and \
table_format not in schema_include_constraints[table_name.lower()]:
- print 'Skipping \'%s.%s\' due to include constraint match.' % (db, table_name)
+ print('Skipping \'%s.%s\' due to include constraint match.' % (db, table_name))
continue
if schema_exclude_constraints[table_name.lower()] and\
table_format in schema_exclude_constraints[table_name.lower()]:
- print 'Skipping \'%s.%s\' due to exclude constraint match.' % (db, table_name)
+ print('Skipping \'%s.%s\' due to exclude constraint match.' % (db, table_name))
continue
alter = section.get('ALTER')
@@ -752,7 +754,7 @@ def generate_statements(output_name, test_vectors, sections,
# TODO: Currently, Kudu does not support partitioned tables via Impala.
# If a CREATE_KUDU section was provided, assume it handles the partition columns
if file_format == 'kudu' and partition_columns != '' and not create_kudu:
- print "Ignore partitions on Kudu table: %s.%s" % (db_name, table_name)
+ print("Ignore partitions on Kudu table: %s.%s" % (db_name, table_name))
continue
# If a CREATE section is provided, use that. Otherwise a COLUMNS section
@@ -817,15 +819,15 @@ def generate_statements(output_name, test_vectors, sections,
# and skip loading the data. Otherwise, the data is generated using either an
# INSERT INTO statement or a LOAD statement.
if not force_reload and hdfs_location in existing_tables:
- print 'HDFS path:', data_path, 'contains data. Data loading can be skipped.'
+ print('HDFS path:', data_path, 'contains data. Data loading can be skipped.')
else:
- print 'HDFS path:', data_path, 'does not exist or is empty. Data will be loaded.'
+ print('HDFS path:', data_path, 'does not exist or is empty. Data will be loaded.')
if not db_suffix:
if load:
hive_output.load_base.append(build_load_statement(load, db_name,
db_suffix, table_name))
else:
- print 'Empty base table load for %s. Skipping load generation' % table_name
+ print('Empty base table load for %s. Skipping load generation' % table_name)
elif file_format in ['kudu', 'parquet', 'iceberg']:
if insert_hive:
hive_output.load.append(build_insert(insert_hive, db_name, db_suffix,
@@ -835,8 +837,8 @@ def generate_statements(output_name, test_vectors, sections,
file_format, codec, compression_type, table_name, data_path,
for_impala=True))
else:
- print 'Empty parquet/kudu load for table %s. Skipping insert generation' \
- % table_name
+ print('Empty parquet/kudu load for table %s. Skipping insert generation'
+ % table_name)
else:
if insert_hive:
insert = insert_hive
@@ -844,7 +846,7 @@ def generate_statements(output_name, test_vectors, sections,
hive_output.load.append(build_insert(insert, db_name, db_suffix, file_format,
codec, compression_type, table_name, data_path, create_hive=create_hive))
else:
- print 'Empty insert for table %s. Skipping insert generation' % table_name
+ print('Empty insert for table %s. Skipping insert generation' % table_name)
impala_create.write_to_file("create-%s-impala-generated-%s-%s-%s.sql" %
(output_name, file_format, codec, compression_type))
@@ -879,8 +881,8 @@ def parse_schema_template_file(file_name):
if __name__ == "__main__":
if options.table_formats is None:
if options.exploration_strategy not in KNOWN_EXPLORATION_STRATEGIES:
- print 'Invalid exploration strategy:', options.exploration_strategy
- print 'Valid values:', ', '.join(KNOWN_EXPLORATION_STRATEGIES)
+ print('Invalid exploration strategy:', options.exploration_strategy)
+ print('Valid values:', ', '.join(KNOWN_EXPLORATION_STRATEGIES))
sys.exit(1)
test_vectors = [vector.value for vector in\
@@ -896,7 +898,7 @@ if __name__ == "__main__":
convert_orc_to_full_acid = options.workload == 'functional-query'
target_dataset = test_vectors[0].dataset
- print 'Target Dataset: ' + target_dataset
+ print('Target Dataset: ' + target_dataset)
dataset_load_dir = os.path.join(SQL_OUTPUT_DIR, target_dataset)
# If the directory containing the sql files does not exist, create it. Else nuke all the
# files corresponding to the current workload.
@@ -918,7 +920,7 @@ if __name__ == "__main__":
'%s_schema_template.sql' % target_dataset)
if not os.path.isfile(schema_template_file):
- print 'Schema file not found: ' + schema_template_file
+ print('Schema file not found: ' + schema_template_file)
sys.exit(1)
constraints_file = os.path.join(DATASET_DIR, target_dataset, 'schema_constraints.csv')
diff --git a/testdata/bin/generate-test-vectors.py b/testdata/bin/generate-test-vectors.py
index 9d0be9e50..4998a8caa 100755
--- a/testdata/bin/generate-test-vectors.py
+++ b/testdata/bin/generate-test-vectors.py
@@ -40,6 +40,7 @@
# The pairwise generation is done using the Python 'AllPairs' module. This module can be
# downloaded from http://pypi.python.org/pypi/AllPairs/2.0.1
#
+from __future__ import print_function
import collections
import csv
import math
@@ -56,7 +57,7 @@ parser.add_option("-w", "--workload", dest="workload",
(options, args) = parser.parse_args()
if options.workload is None:
- print "A workload name must be specified."
+ print("A workload name must be specified.")
parser.print_help()
sys.exit(1)
@@ -115,12 +116,12 @@ def read_dimension_file(file_name):
continue
values = line.split(':')
if len(values) != 2:
- print 'Invalid dimension file format. Expected format is <dimension name>: val1,'\
- ' val2, ... Found: ' + line
+ print('Invalid dimension file format. Expected format is <dimension name>: val1,'
+ ' val2, ... Found: ' + line)
sys.exit(1)
if not values[0] in KNOWN_DIMENSION_NAMES:
- print 'Unknown dimension name: ' + values[0]
- print 'Valid dimension names: ' + ', '.join(KNOWN_DIMENSION_NAMES)
+ print('Unknown dimension name: ' + values[0])
+ print('Valid dimension names: ' + ', '.join(KNOWN_DIMENSION_NAMES))
sys.exit(1)
dimension_map[values[0]] = [val.strip() for val in values[1].split(',')]
return dimension_map
@@ -132,7 +133,7 @@ def write_vectors_to_csv(output_dir, output_file, matrix):
output_text += '\n' + ', '.join(row)
output_path = os.path.join(output_dir, output_file)
- print 'Writing test vectors to: ' + output_path
+ print('Writing test vectors to: ' + output_path)
with open(output_path, 'wb') as output_file:
output_file.write(output_text)
output_file.write('\n')
@@ -140,10 +141,10 @@ def write_vectors_to_csv(output_dir, output_file, matrix):
dimension_file = os.path.join(WORKLOAD_DIR, options.workload,
'%s_dimensions.csv' % options.workload)
if not os.path.isfile(dimension_file):
- print 'Dimension file not found: ' + dimension_file
+ print('Dimension file not found: ' + dimension_file)
sys.exit(1)
-print 'Reading dimension file: ' + dimension_file
+print('Reading dimension file: ' + dimension_file)
vector_map = read_dimension_file(dimension_file)
vectors = []
diff --git a/testdata/bin/load-tpc-kudu.py b/testdata/bin/load-tpc-kudu.py
index 01de79ee6..422d064ba 100755
--- a/testdata/bin/load-tpc-kudu.py
+++ b/testdata/bin/load-tpc-kudu.py
@@ -22,6 +22,7 @@
# Kudu tables are created in the specified 'target-db' using the existing HDFS tables
# from 'source-db'.
+from __future__ import print_function
import logging
import os
import sqlparse
@@ -72,7 +73,7 @@ def load_data():
query = sqlparse.format(query.rstrip(';'), strip_comments=True)
query_str = query.format(**sql_params)
if (len(query_str)) == 0: continue
- if verbose: print query_str
+ if verbose: print(query_str)
impala.execute(query_str)
def get_test_file_path(workload):
diff --git a/testdata/bin/rewrite-iceberg-metadata.py b/testdata/bin/rewrite-iceberg-metadata.py
index 1ccee49a7..26997345b 100755
--- a/testdata/bin/rewrite-iceberg-metadata.py
+++ b/testdata/bin/rewrite-iceberg-metadata.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import glob
import json
import os
diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py
index bf57ea89d..6e99fa5f3 100755
--- a/testdata/bin/wait-for-hiveserver2.py
+++ b/testdata/bin/wait-for-hiveserver2.py
@@ -22,6 +22,7 @@
# TODO: Consider combining this with wait-for-metastore.py. A TCLIService client
# can perhaps also talk to the metastore.
+from __future__ import print_function
import os
import time
import getpass
@@ -71,20 +72,20 @@ while time.time() - now < TIMEOUT_SECONDS:
close_session_req = TCLIService.TCloseSessionReq()
close_session_req.sessionHandle = resp.sessionHandle
hs2_client.CloseSession(close_session_req)
- print "HiveServer2 service is up at %s." % options.hs2_hostport
+ print("HiveServer2 service is up at %s." % options.hs2_hostport)
exit(0)
except Exception as e:
if "SASL" in e.message: # Bail out on SASL failures
- print "SASL failure when attempting connection:"
+ print("SASL failure when attempting connection:")
raise
if "GSS" in e.message: # Other GSSAPI failures
- print "GSS failure when attempting connection:"
+ print("GSS failure when attempting connection:")
raise
- print "Waiting for HiveServer2 at %s..." % options.hs2_hostport
- print e
+ print("Waiting for HiveServer2 at %s..." % options.hs2_hostport)
+ print(e)
finally:
hs2_transport.close()
time.sleep(0.5)
-print "HiveServer2 service failed to start within %s seconds." % TIMEOUT_SECONDS
+print("HiveServer2 service failed to start within %s seconds." % TIMEOUT_SECONDS)
exit(1)
diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py
index 2dc6550dd..0707cc16b 100755
--- a/testdata/bin/wait-for-metastore.py
+++ b/testdata/bin/wait-for-metastore.py
@@ -21,6 +21,7 @@
# to execute the get_database("default") Thrift RPC until the call succeeds,
# or a timeout is reached.
+from __future__ import print_function
import os
import time
from optparse import OptionParser
@@ -56,19 +57,19 @@ while time.time() - now < TIMEOUT_SECONDS:
hive_transport.open()
resp = hive_client.get_database("default")
if resp is not None:
- print "Metastore service is up at %s." % options.metastore_hostport
+ print("Metastore service is up at %s." % options.metastore_hostport)
exit(0)
except Exception as e:
if "SASL" in e.message: # Bail out on SASL failures
- print "SASL failure when attempting connection:"
+ print("SASL failure when attempting connection:")
raise
if "GSS" in e.message: # Other GSSAPI failures
- print "GSS failure when attempting connection:"
+ print("GSS failure when attempting connection:")
raise
- print "Waiting for the Metastore at %s..." % options.metastore_hostport
+ print("Waiting for the Metastore at %s..." % options.metastore_hostport)
finally:
hive_transport.close()
time.sleep(0.5)
-print "Metastore service failed to start within %s seconds." % TIMEOUT_SECONDS
+print("Metastore service failed to start within %s seconds." % TIMEOUT_SECONDS)
exit(1)
diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py
index 73f7edc1d..00443a71a 100644
--- a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py
+++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import os
import sys
@@ -39,7 +40,7 @@ def _get_yarn_nm_ram_mb():
# - leave at least 20G for other services
# - don't need more than 48G
ret = min(max(available_ram_gb * 1024 - 20 * 1024, 4096), 48 * 1024)
- print >>sys.stderr, "Configuring Yarn NM to use {0}MB RAM".format(ret)
+ print("Configuring Yarn NM to use {0}MB RAM".format(ret), file=sys.stderr)
return ret
diff --git a/testdata/common/cgroups.py b/testdata/common/cgroups.py
index c313b563f..5e1f6f73e 100755
--- a/testdata/common/cgroups.py
+++ b/testdata/common/cgroups.py
@@ -19,6 +19,7 @@
# Utility code for creating cgroups for the Impala development environment.
# May be used as a library or as a command-line utility for manual testing.
+from __future__ import print_function
import os
import sys
import errno
@@ -81,7 +82,7 @@ def create_impala_cgroup_path(instance_num):
if __name__ == "__main__":
if options.cluster_size < 0:
- print 'Please specify a cluster size >= 0'
+ print('Please specify a cluster size >= 0')
sys.exit(1)
for i in range(options.cluster_size):
create_impala_cgroup_path(i)
diff --git a/testdata/common/text_delims_table.py b/testdata/common/text_delims_table.py
index b51441b67..92b2d01a4 100755
--- a/testdata/common/text_delims_table.py
+++ b/testdata/common/text_delims_table.py
@@ -22,6 +22,7 @@
# command line, will generate data files in the specified directory and a
# print a SQL load statement to incorporate into dataload SQL script generation.
+from __future__ import print_function
from shutil import rmtree
from optparse import OptionParser
from contextlib import contextmanager
diff --git a/testdata/common/widetable.py b/testdata/common/widetable.py
index d83c62e00..dcd31de49 100755
--- a/testdata/common/widetable.py
+++ b/testdata/common/widetable.py
@@ -22,6 +22,7 @@
# generate a CSV data file and prints a SQL load statement to incorporate
# into dataload SQL script generation.
+from __future__ import print_function
from datetime import datetime, timedelta
import itertools
import optparse
@@ -119,7 +120,7 @@ if __name__ == "__main__":
if options.get_columns:
# Output column descriptors
- print '\n'.join(get_columns(options.num_columns))
+ print('\n'.join(get_columns(options.num_columns)))
if options.create_data:
# Generate data locally, and output the SQL load command for use in dataload
diff --git a/tests/benchmark/report_benchmark_results.py b/tests/benchmark/report_benchmark_results.py
index f8a210568..7ce8ba688 100755
--- a/tests/benchmark/report_benchmark_results.py
+++ b/tests/benchmark/report_benchmark_results.py
@@ -29,6 +29,7 @@
# if necessary (2.5).
from __future__ import division
+from __future__ import print_function
import difflib
import json
import logging
@@ -1150,5 +1151,5 @@ if __name__ == "__main__":
if ref_grouped:
ref_impala_version = get_impala_version(ref_grouped)
- print build_summary_header(current_impala_version, ref_impala_version)
- print report
+ print(build_summary_header(current_impala_version, ref_impala_version))
+ print(report)
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 8e0860fec..4a0ccb940 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -17,6 +17,7 @@
#
# Basic object model of an Impala cluster (set of Impala processes).
+from __future__ import print_function
import json
import logging
import os
@@ -158,7 +159,7 @@ class ImpalaCluster(object):
result = client.execute("select 1")
assert result.success
++n
- except Exception as e: print e
+ except Exception as e: print(e)
finally:
client.close()
return n
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 98d7ba164..033b0c4f4 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -17,6 +17,7 @@
#
# The base class that should be used for almost all Impala tests
+from __future__ import print_function
import glob
import grp
import json
@@ -1202,7 +1203,7 @@ class ImpalaTestSuite(BaseTestSuite):
db_name=db_name, table_name=table_name))
return
except Exception as ex:
- print str(ex)
+ print(str(ex))
time.sleep(0.2)
continue
raise Exception("Table {0}.{1} didn't show up after {2}s", db_name, table_name,
diff --git a/tests/common/resource_pool_config.py b/tests/common/resource_pool_config.py
index 9347b7e9d..88f9b55e4 100644
--- a/tests/common/resource_pool_config.py
+++ b/tests/common/resource_pool_config.py
@@ -20,6 +20,7 @@
# the tests it is used for. However, it is generic enough that it can be extended if
# more functionality is required for adding tests.
+from __future__ import print_function
import os
from time import sleep, time
import xml.etree.ElementTree as ET
@@ -98,7 +99,7 @@ class ResourcePoolConfig(object):
if pool_name == name.split('.')[-1] and pool_attribute in name:
return property
except Exception as e:
- print "Current DOM element being inspected: \n{0}".format(ET.dump(property))
+ print("Current DOM element being inspected: \n{0}".format(ET.dump(property)))
raise e
assert False, "{0} attribute not found for pool {1} in the config XML:\n{2}".format(
pool_attribute, pool_name, ET.dump(xml_root))
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index f3a3c5091..81fe672b1 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -16,6 +16,7 @@
# under the License.
"""This module provides utilities for interacting with a cluster."""
+from __future__ import print_function
# This should be moved into the test/util folder eventually. The problem is this
# module depends on db_connection which use some query generator classes.
diff --git a/tests/comparison/data_generator_mapper.py b/tests/comparison/data_generator_mapper.py
index 897494035..12135f563 100755
--- a/tests/comparison/data_generator_mapper.py
+++ b/tests/comparison/data_generator_mapper.py
@@ -20,6 +20,7 @@
'''This is a mapper for use with hadoop streaming. See data_generator.DatabasePopulator
for more information on how this file is used.
'''
+from __future__ import print_function
import os
import random
diff --git a/tests/comparison/db_connection.py b/tests/comparison/db_connection.py
index 71786b83d..a70a896c3 100644
--- a/tests/comparison/db_connection.py
+++ b/tests/comparison/db_connection.py
@@ -21,6 +21,7 @@
connection.
'''
+from __future__ import print_function
import hashlib
import impala.dbapi
import re
diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py
index d7a792a16..7a223db8f 100755
--- a/tests/comparison/discrepancy_searcher.py
+++ b/tests/comparison/discrepancy_searcher.py
@@ -22,6 +22,7 @@
results.
'''
+from __future__ import print_function
# TODO: IMPALA-4600: refactor this module
from copy import deepcopy
diff --git a/tests/comparison/query_generator.py b/tests/comparison/query_generator.py
index 7fe998278..361c1cb46 100644
--- a/tests/comparison/query_generator.py
+++ b/tests/comparison/query_generator.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
from collections import defaultdict
from copy import deepcopy
from itertools import ifilter
diff --git a/tests/comparison/util/verify-oracle-connection.py b/tests/comparison/util/verify-oracle-connection.py
index 3db4ed128..f4435b40f 100755
--- a/tests/comparison/util/verify-oracle-connection.py
+++ b/tests/comparison/util/verify-oracle-connection.py
@@ -31,6 +31,7 @@
# Importing the whole module instead of doing selective import seems to help find linker
# errors.
+from __future__ import print_function
import cx_Oracle
# Host on which Oracle Database lies.
@@ -56,7 +57,7 @@ def main():
finally:
cursor.close()
assert rows == EXPECTED_RESULT
- print 'success'
+ print('success')
if '__main__' == __name__:
diff --git a/tests/conftest.py b/tests/conftest.py
index 761f0cb73..0616e9110 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -17,6 +17,7 @@
# py.test configuration module
#
+from __future__ import print_function
from impala.dbapi import connect as impala_connect
from kudu import connect as kudu_connect
from random import choice, sample
diff --git a/tests/custom_cluster/test_blacklist.py b/tests/custom_cluster/test_blacklist.py
index 7fd61d891..20b18b508 100644
--- a/tests/custom_cluster/test_blacklist.py
+++ b/tests/custom_cluster/test_blacklist.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
import pytest
@@ -226,7 +227,7 @@ class TestBlacklistFaultyDisk(CustomClusterTestSuite):
dir_path = tempfile.mkdtemp()
self.created_dirs.append(dir_path)
result.append(dir_path)
- print "Generated dir" + dir_path
+ print("Generated dir" + dir_path)
return result
def setup_method(self, method):
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index d106093ef..aee47772c 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import glob
import os
import psutil
@@ -89,7 +90,7 @@ class TestBreakpadBase(CustomClusterTestSuite):
# For every process in the list we might see the original Impala process plus a
# forked off child that is writing the minidump. We need to catch both.
for pid in process.get_pids():
- print "Checking pid %s" % pid
+ print("Checking pid %s" % pid)
psutil_process = psutil.Process(pid)
psutil_process.wait(timeout)
except psutil.NoSuchProcess:
diff --git a/tests/custom_cluster/test_catalog_hms_failures.py b/tests/custom_cluster/test_catalog_hms_failures.py
index 4d879852b..78c52f42b 100644
--- a/tests/custom_cluster/test_catalog_hms_failures.py
+++ b/tests/custom_cluster/test_catalog_hms_failures.py
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import pytest
import os
import time
@@ -75,7 +76,7 @@ class TestHiveMetaStoreFailure(CustomClusterTestSuite):
try:
self.client.execute("describe %s" % tbl_name)
except ImpalaBeeswaxException as e:
- print str(e)
+ print(str(e))
assert "Failed to load metadata for table: %s. Running 'invalidate metadata %s' "\
"may resolve this problem." % (tbl_name, tbl_name) in str(e)
self.run_hive_server()
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index c368cc4d0..b2a5267d3 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -16,6 +16,7 @@
# under the License.
#
+from __future__ import print_function
import json
import logging
import os
@@ -128,7 +129,7 @@ class TestClientSsl(CustomClusterTestSuite):
p.send_cmd("profile")
result = p.get_result()
- print result.stderr
+ print(result.stderr)
assert "Query Status: Cancelled" in result.stdout
assert impalad.wait_for_num_in_flight_queries(0)
diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py
index e8a8f88fc..36bc7ebdf 100644
--- a/tests/custom_cluster/test_events_custom_configs.py
+++ b/tests/custom_cluster/test_events_custom_configs.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import logging
import pytest
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 78eccb4ae..5f7a25332 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -17,6 +17,7 @@
# Test behaviors specific to --use_local_catalog being enabled.
+from __future__ import print_function
import pytest
import Queue
import random
@@ -272,7 +273,7 @@ class TestLocalCatalogRetries(CustomClusterTestSuite):
q = random.choice(queries)
attempt += 1
try:
- print 'Attempt', attempt, 'client', str(client)
+ print('Attempt', attempt, 'client', str(client))
ret = self.execute_query_unchecked(client, q)
except Exception as e:
if 'InconsistentMetadataFetchException' in str(e):
diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py
index 1a5f5efdb..d7e8e4fb7 100644
--- a/tests/custom_cluster/test_query_retries.py
+++ b/tests/custom_cluster/test_query_retries.py
@@ -21,6 +21,7 @@
# TODO: Re-factor tests into multiple classes.
# TODO: Add a test that cancels queries while a retry is running
+from __future__ import print_function
import pytest
import re
import shutil
@@ -1183,7 +1184,7 @@ class TestQueryRetriesFaultyDisk(CustomClusterTestSuite):
dir_path = tempfile.mkdtemp()
self.created_dirs.append(dir_path)
result.append(dir_path)
- print "Generated dir" + dir_path
+ print("Generated dir" + dir_path)
return result
def __validate_web_ui_state(self):
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index fb1564943..9740974bb 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import logging
import os
import pytest
@@ -515,7 +516,7 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
def test_shutdown_executor_with_delay(self):
"""Regression test for IMPALA-7931 that adds delays to status reporting and
to fetching of results to trigger races that previously resulted in query failures."""
- print self.exploration_strategy
+ print(self.exploration_strategy)
if self.exploration_strategy() != 'exhaustive':
pytest.skip()
self.do_test_shutdown_executor(fetch_delay_s=5)
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 35adf1c40..4ed4ebba5 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -17,6 +17,7 @@
#
# Tests for query expiration.
+from __future__ import print_function
import os
import pytest
import re
@@ -82,7 +83,7 @@ class TestScratchDir(CustomClusterTestSuite):
if not non_existing:
self.created_dirs.append(dir_path)
result.append(dir_path)
- print "Generated dir" + dir_path
+ print("Generated dir" + dir_path)
return result
def setup_method(self, method):
diff --git a/tests/custom_cluster/test_topic_update_frequency.py b/tests/custom_cluster/test_topic_update_frequency.py
index 99d7b659c..13402714e 100644
--- a/tests/custom_cluster/test_topic_update_frequency.py
+++ b/tests/custom_cluster/test_topic_update_frequency.py
@@ -9,6 +9,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import print_function
from multiprocessing.pool import ThreadPool
import pytest
diff --git a/tests/custom_cluster/test_udf_concurrency.py b/tests/custom_cluster/test_udf_concurrency.py
index 8643ecf57..8e002655d 100644
--- a/tests/custom_cluster/test_udf_concurrency.py
+++ b/tests/custom_cluster/test_udf_concurrency.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import os
import pytest
import random
@@ -82,7 +83,7 @@ class TestUdfConcurrency(CustomClusterTestSuite):
try:
setup_client.execute(setup_query)
except Exception as e:
- print "Unable to create initial function: {0}".format(setup_query)
+ print("Unable to create initial function: {0}".format(setup_query))
raise
errors = []
@@ -126,7 +127,7 @@ class TestUdfConcurrency(CustomClusterTestSuite):
# join all threads.
for t in runner_threads: t.join()
- for e in errors: print e
+ for e in errors: print(e)
# Checks that no impalad has crashed.
assert cluster.num_responsive_coordinators() == exp_num_coordinators
@@ -167,13 +168,13 @@ class TestUdfConcurrency(CustomClusterTestSuite):
s = create_fn_to_use.format(unique_database, udf_tgt_path)
setup_client.execute(s)
except Exception as e:
- print e
+ print(e)
assert False
for i in range(0, num_drops):
try:
setup_client.execute(create_fn_to_drop.format(unique_database, i, udf_tgt_path))
except Exception as e:
- print e
+ print(e)
assert False
errors = []
@@ -208,5 +209,5 @@ class TestUdfConcurrency(CustomClusterTestSuite):
for t in runner_threads: t.join()
# Check for any errors.
- for e in errors: print e
+ for e in errors: print(e)
assert len(errors) == 0
diff --git a/tests/custom_cluster/test_web_pages.py b/tests/custom_cluster/test_web_pages.py
index 687f6d054..fb983cb9d 100644
--- a/tests/custom_cluster/test_web_pages.py
+++ b/tests/custom_cluster/test_web_pages.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import json
import random
import re
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 05909089a..828033a9f 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -23,7 +23,7 @@
# TODO: For each test, verify all the metadata available via Hive and
# Impala, in all the possible ways of validating that metadata.
-
+from __future__ import print_function
import pytest
import random
import string
@@ -735,7 +735,7 @@ class TestHmsIntegration(ImpalaTestSuite):
'(x int, y int) stored as parquet') as table_name:
self.client.execute('invalidate metadata')
self.client.execute('invalidate metadata %s' % table_name)
- print self.impala_table_stats(table_name)
+ print(self.impala_table_stats(table_name))
assert 'PARQUET' == self.impala_table_stats(table_name)[()]['format']
self.run_stmt_in_hive(
'alter table %s set fileformat avro' % table_name)
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index 5e450485e..165e595ce 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -17,6 +17,7 @@
# Validates all aggregate functions across all datatypes
#
+from __future__ import print_function
import pytest
from testdata.common import widetable
@@ -150,7 +151,8 @@ class TestAggregation(ImpalaTestSuite):
# NDV is inherently approximate. Compare with some tolerance.
err = abs(result_lut[key] - int(actual_string))
rel_err = err / float(result_lut[key])
- print key, result_lut[key], actual_string,abs(result_lut[key] - int(actual_string))
+ print(key, result_lut[key], actual_string,
+ abs(result_lut[key] - int(actual_string)))
assert err <= 1 or rel_err < 0.05
elif data_type in ('float', 'double') and agg_func != 'count':
# Compare with a margin of error.
diff --git a/tests/query_test/test_compressed_formats.py b/tests/query_test/test_compressed_formats.py
index 5d8bb17b7..e9187445d 100644
--- a/tests/query_test/test_compressed_formats.py
+++ b/tests/query_test/test_compressed_formats.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import math
import os
import pytest
@@ -96,7 +97,7 @@ class TestCompressedFormatsBase(ImpalaTestSuite):
assert result and int(result) > 0
except Exception as e:
error_msg = str(e)
- print error_msg
+ print(error_msg)
if expected_error is None or expected_error not in error_msg:
print("Unexpected error:\n{0}".format(error_msg))
raise
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 4f9f82c4a..32f91702a 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -17,6 +17,7 @@
# Validates limit on scan nodes
+from __future__ import print_function
import pytest
import re
import time
@@ -87,7 +88,7 @@ class TestHdfsCaching(ImpalaTestSuite):
elif num_metrics_increased != 1:
# Test failed, print the metrics
for i in range(0, len(cached_bytes_before)):
- print "%d %d" % (cached_bytes_before[i], cached_bytes_after[i])
+ print("%d %d" % (cached_bytes_before[i], cached_bytes_after[i]))
assert(False)
def test_cache_cancellation(self, vector):
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 587431dd0..0f1c59a17 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
from kudu.schema import (
BOOL,
DOUBLE,
@@ -481,7 +482,7 @@ class TestKuduOperations(KuduTestSuite):
date(2010, 1, 1), '')]
i += 1
cursor.execute("select count(*) from %s" % table_name)
- print cursor.fetchall() == [(i, )]
+ print(cursor.fetchall() == [(i, )])
def test_concurrent_schema_change(self, cursor, unique_database):
"""Tests that an insert into a Kudu table with a concurrent schema change either
diff --git a/tests/query_test/test_limit.py b/tests/query_test/test_limit.py
index 322429906..5233b4682 100644
--- a/tests/query_test/test_limit.py
+++ b/tests/query_test/test_limit.py
@@ -17,6 +17,7 @@
# Validates limit on scan nodes
+from __future__ import print_function
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_vector import ImpalaTestDimension
@@ -79,5 +80,5 @@ class TestLimitBase(ImpalaTestSuite):
except ImpalaBeeswaxException as e:
assert not should_succeed, 'Query was not expected to fail: %s' % e
if (expected_error not in str(e)):
- print str(e)
+ print(str(e))
assert expected_error in str(e)
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 5298ab152..be5a47578 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -21,6 +21,7 @@
# tests can run with the normal exploration strategy and the overall test runtime doesn't
# explode.
+from __future__ import print_function
import os
import pytest
import random
@@ -410,10 +411,10 @@ class TestHdfsScannerSkew(ImpalaTestSuite):
ratio = float(a) / float(b)
else:
ratio = float(b) / float(a)
- print "Intra-node bytes read ratio:", ratio
+ print("Intra-node bytes read ratio:", ratio)
if ratio < SKEW_THRESHOLD:
count_skew += 1
- print "Profile of skewed execution: ", profile
+ print("Profile of skewed execution: ", profile)
return count_skew
tbl_name = unique_database + ".lineitem_skew"
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 0ddabff4b..7c1ae8559 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -22,6 +22,7 @@
# executing the remaining tests in parallel. To run only some of
# these, use --skip-serial, --skip-stress, or --skip-parallel.
# All additional command line options are passed to py.test.
+from __future__ import print_function
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_service import ImpaladService
from tests.conftest import configure_logging
@@ -226,16 +227,16 @@ def build_ignore_dir_arg_list(valid_dirs):
def print_metrics(substring):
"""Prints metrics with the given substring in the name"""
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
- print ">" * 80
+ print(">" * 80)
port = impalad.get_webserver_port()
cert = impalad._get_webserver_certificate_file()
- print "connections metrics for impalad at port {0}:".format(port)
+ print("connections metrics for impalad at port {0}:".format(port))
debug_info = json.loads(ImpaladService(impalad.hostname, webserver_port=port,
webserver_certificate_file=cert).read_debug_webpage('metrics?json'))
for metric in debug_info['metric_group']['metrics']:
if substring in metric['name']:
- print json.dumps(metric, indent=1)
- print "<" * 80
+ print(json.dumps(metric, indent=1))
+ print("<" * 80)
def detect_and_remove_flag(flag):
@@ -265,7 +266,7 @@ if __name__ == "__main__":
def run(args):
"""Helper to print out arguments of test_executor before invoking."""
- print "Running TestExecutor with args: %s" % (args,)
+ print("Running TestExecutor with args: %s" % (args,))
test_executor.run_tests(args)
os.chdir(TEST_DIR)
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index d957db135..30ae99f16 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -18,6 +18,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import httplib
import logging
import os
@@ -896,14 +897,14 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
os.chdir("%s/tests/shell/" % os.environ['IMPALA_HOME'])
result = run_impala_shell_interactive(vector,
"sOuRcE shell_case_sensitive.cmds; SeLeCt 'second command'")
- print result.stderr
+ print(result.stderr)
assert "Query: uSe FUNCTIONAL" in result.stderr
assert "Query: ShOw TABLES" in result.stderr
assert "alltypes" in result.stdout
# This is from shell_case_sensitive2.cmds, the result of sourcing a file
# from a sourced file.
- print result.stderr
+ print(result.stderr)
assert "SeLeCt 'second command'" in result.stderr
finally:
os.chdir(cwd)
diff --git a/tests/shell/util.py b/tests/shell/util.py
index 98982d126..f369c646d 100755
--- a/tests/shell/util.py
+++ b/tests/shell/util.py
@@ -18,6 +18,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import logging
import os
import socket
diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py
index 9daedba30..5dfead822 100644
--- a/tests/statestore/test_statestore.py
+++ b/tests/statestore/test_statestore.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
from collections import defaultdict
import json
import logging
@@ -155,7 +156,7 @@ class KillableThreadedServer(TServer):
except TTransport.TTransportException as tx:
pass
except Exception as x:
- print x
+ print(x)
itrans.close()
otrans.close()
diff --git a/tests/stress/extract_min_mem.py b/tests/stress/extract_min_mem.py
index e84148ba3..5749d164f 100755
--- a/tests/stress/extract_min_mem.py
+++ b/tests/stress/extract_min_mem.py
@@ -33,6 +33,7 @@
#
# ./tests/stress/extract_min_mem.py mem_usage_scaling_runtime_info.json
#
+from __future__ import print_function
import json
import sys
@@ -46,4 +47,4 @@ with open(sys.argv[1]) as f:
runtime_info['required_mem_mb_with_spilling']))
results.sort()
-print ', '.join(["'Q{0}': {1}".format(num, mem) for num, mem in results])
+print(', '.join(["'Q{0}': {1}".format(num, mem) for num, mem in results]))
diff --git a/tests/stress/runtime_info.py b/tests/stress/runtime_info.py
index 6cba38590..d9ac7c757 100644
--- a/tests/stress/runtime_info.py
+++ b/tests/stress/runtime_info.py
@@ -20,6 +20,7 @@
# Utility functions used by the stress test to save and load runtime info
# about queries to and from JSON files.
+from __future__ import print_function
from collections import defaultdict
import json
import logging
diff --git a/tests/unittests/test_file_parser.py b/tests/unittests/test_file_parser.py
index 54a00f709..5e79eae1e 100644
--- a/tests/unittests/test_file_parser.py
+++ b/tests/unittests/test_file_parser.py
@@ -17,6 +17,7 @@
# Unit tests for the test file parser
+from __future__ import print_function
from tests.common.base_test_suite import BaseTestSuite
from tests.util.test_file_parser import parse_test_file_text
@@ -66,7 +67,7 @@ class TestTestFileParser(BaseTestSuite):
def test_valid_parse(self):
results = parse_test_file_text(test_text, VALID_SECTIONS)
assert len(results) == 3
- print results[0]
+ print(results[0])
expected_results = {'QUERY': '# comment\nSELECT blah from Foo\ns\n',
'TYPES': 'string\n', 'RESULTS': "'Hi'\n",
'LINEAGE': "test_lineage_str > 'foo' AND 'bar'\nmulti_line\n"}
@@ -106,6 +107,6 @@ class TestTestFileParser(BaseTestSuite):
"#---- QUERY: TEST_WORKLOAD_Q2\n"
"#SELECT int_col from Bar\n"
"#---- RESULTS\n#231\n#---- TYPES\n#int\n"}
- print expected_results
- print results[1]
+ print(expected_results)
+ print(results[1])
assert results[1] == expected_results
diff --git a/tests/util/acid_txn.py b/tests/util/acid_txn.py
index c50f4572f..0bc82125b 100644
--- a/tests/util/acid_txn.py
+++ b/tests/util/acid_txn.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
from tests.util.thrift_util import create_transport
from hive_metastore import ThriftHiveMetastore
from hive_metastore.ttypes import (AbortTxnRequest, AllocateTableWriteIdsRequest,
@@ -148,4 +149,4 @@ class AcidTxn(object):
try:
self.commit_txn(txn)
except Exception as e:
- print str(e)
+ print(str(e))
diff --git a/tests/util/run_impyla_http_query.py b/tests/util/run_impyla_http_query.py
index 39b1ba379..df81f8793 100755
--- a/tests/util/run_impyla_http_query.py
+++ b/tests/util/run_impyla_http_query.py
@@ -20,6 +20,7 @@
# It can be used by other tests (e.g. LdapImpylaHttpTest.java) that start a cluster with
# an LDAP server to validate Impyla's functionality.
+from __future__ import print_function
import argparse
import logging
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index 37e42652d..e94e290da 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -17,6 +17,7 @@
# This module is used for common utilities related to parsing test files
+from __future__ import print_function
import codecs
import collections
import logging
@@ -210,8 +211,8 @@ def parse_test_file_text(text, valid_section_names, skip_unknown_sections=True):
if subsection_name not in valid_section_names:
if skip_unknown_sections or not subsection_name:
- print sub_section
- print 'Unknown section \'%s\'' % subsection_name
+ print(sub_section)
+ print('Unknown section \'%s\'' % subsection_name)
continue
else:
raise RuntimeError, 'Unknown subsection: %s' % subsection_name