You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2020/01/16 04:39:54 UTC
[kudu] branch master updated (31ed4a1 -> 8463663)
This is an automated email from the ASF dual-hosted git repository.
adar pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from 31ed4a1 KUDU-3011 p6: don't transfer leadership to quiescing followers
new ad8fc53 [build] Fix the RAT report
new 8463663 [python] Fix Python 3 syntax issues
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build-support/build_source_release.py | 2 +-
build-support/check_compatibility.py | 4 ++--
build-support/clang_tidy_gerrit.py | 25 +++++++++++-----------
build-support/dist_test.py | 6 +++---
build-support/iwyu.py | 2 +-
.../relocate_binaries_for_mini_cluster.py | 6 +++---
build-support/parse_test_failure.py | 12 +++++++----
build-support/push_to_asf.py | 2 +-
build-support/release/check-rat-report.py | 2 +-
build-support/release/rat_exclude_files.txt | 2 ++
build-support/run_dist_test.py | 4 ++--
build-support/test_result_server.py | 6 +++++-
.../python/graphite-kudu/kudu/kudu_graphite.py | 2 +-
src/kudu/benchmarks/wal_hiccup-parser.py | 6 +++---
src/kudu/experiments/merge-test.py | 4 ++++
src/kudu/scripts/get-job-stats-from-mysql.py | 4 ++--
src/kudu/scripts/graph-metrics.py | 4 ++--
src/kudu/scripts/max_skew_estimate.py | 8 +++++--
src/kudu/scripts/parse_metrics_log.py | 10 ++++-----
src/kudu/scripts/write-jobs-stats-to-mysql.py | 2 +-
20 files changed, 65 insertions(+), 48 deletions(-)
[kudu] 01/02: [build] Fix the RAT report
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit ad8fc5316ad1459509fa3433d0798c22ecf642dc
Author: Grant Henke <gr...@apache.org>
AuthorDate: Wed Jan 15 14:12:40 2020 -0600
[build] Fix the RAT report
The patch fixes the RAT report by excluding license checks for the
testdata files.
Change-Id: I453f6fcd6e1dd29fa7fc2485db9e7a13879d6f9b
Reviewed-on: http://gerrit.cloudera.org:8080/15038
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Attila Bukor <ab...@apache.org>
Tested-by: Kudu Jenkins
---
build-support/release/rat_exclude_files.txt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/build-support/release/rat_exclude_files.txt b/build-support/release/rat_exclude_files.txt
index 79e4a62..9401276 100644
--- a/build-support/release/rat_exclude_files.txt
+++ b/build-support/release/rat_exclude_files.txt
@@ -183,6 +183,8 @@ src/kudu/util/debug/trace_event_impl.h
src/kudu/util/debug/trace_event_impl_constants.cc
src/kudu/util/debug/trace_event_synthetic_delay.cc
src/kudu/util/debug/trace_event_synthetic_delay.h
+src/kudu/util/testdata/char_truncate_ascii.txt
+src/kudu/util/testdata/char_truncate_utf8.txt
thirdparty/patches/*.patch
www/bootstrap/css/bootstrap-table.min.css
www/bootstrap/css/bootstrap.min.css
[kudu] 02/02: [python] Fix Python 3 syntax issues
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 8463663ca9d6b0f4440dc2d4d321de62b7b6f909
Author: Grant Henke <gr...@apache.org>
AuthorDate: Wed Jan 15 10:23:35 2020 -0600
[python] Fix Python 3 syntax issues
This patch fixes many of the Python 3 incompatibilities in
the various Kudu Python scripts while still maintiaining Python
2 compatibility.
Change-Id: I5d5e06ea6299032d84a76440afb42a70fd461abd
Reviewed-on: http://gerrit.cloudera.org:8080/15037
Reviewed-by: Bankim Bhavsar <ba...@cloudera.com>
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Adar Dembo <ad...@cloudera.com>
---
build-support/build_source_release.py | 2 +-
build-support/check_compatibility.py | 4 ++--
build-support/clang_tidy_gerrit.py | 25 +++++++++++-----------
build-support/dist_test.py | 6 +++---
build-support/iwyu.py | 2 +-
.../relocate_binaries_for_mini_cluster.py | 6 +++---
build-support/parse_test_failure.py | 12 +++++++----
build-support/push_to_asf.py | 2 +-
build-support/release/check-rat-report.py | 2 +-
build-support/run_dist_test.py | 4 ++--
build-support/test_result_server.py | 6 +++++-
.../python/graphite-kudu/kudu/kudu_graphite.py | 2 +-
src/kudu/benchmarks/wal_hiccup-parser.py | 6 +++---
src/kudu/experiments/merge-test.py | 4 ++++
src/kudu/scripts/get-job-stats-from-mysql.py | 4 ++--
src/kudu/scripts/graph-metrics.py | 4 ++--
src/kudu/scripts/max_skew_estimate.py | 8 +++++--
src/kudu/scripts/parse_metrics_log.py | 10 ++++-----
src/kudu/scripts/write-jobs-stats-to-mysql.py | 2 +-
19 files changed, 63 insertions(+), 48 deletions(-)
diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py
index ad53316..cea4104 100755
--- a/build-support/build_source_release.py
+++ b/build-support/build_source_release.py
@@ -187,7 +187,7 @@ def main():
run_rat(tarball_path)
print(Colors.GREEN + "Release successfully generated!" + Colors.RESET)
- print
+ print()
if __name__ == "__main__":
diff --git a/build-support/check_compatibility.py b/build-support/check_compatibility.py
index 61d5661..fbf79ea 100755
--- a/build-support/check_compatibility.py
+++ b/build-support/check_compatibility.py
@@ -162,9 +162,9 @@ def run_java_acc(src_name, src, dst_name, dst):
"\n".join(dst_jars))
annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
- with file(annotations_path, "w") as f:
+ with open(annotations_path, "w") as f:
for ann in PUBLIC_ANNOTATIONS:
- print >>f, ann
+ print(ann, file=f)
java_acc_path = os.path.join(get_java_acc_dir(), "japi-compliance-checker.pl")
diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py
index c10a0e0..d7d1644 100755
--- a/build-support/clang_tidy_gerrit.py
+++ b/build-support/clang_tidy_gerrit.py
@@ -143,10 +143,10 @@ def post_comments(revision_url_base, gerrit_json_obj):
auth=(GERRIT_USER, GERRIT_PASSWORD),
data=json.dumps(gerrit_json_obj),
headers={'Content-Type': 'application/json'})
- print "Response:"
- print r.headers
- print r.status_code
- print r.text
+ print("Response:")
+ print(r.headers)
+ print(r.status_code)
+ print(r.text)
class TestClangTidyGerrit(unittest.TestCase):
@@ -182,7 +182,6 @@ No relevant changes found.
self.assertEqual("src/kudu/blah.cc", parsed[1]['path'])
-
if __name__ == "__main__":
# Basic setup and argument parsing.
init_logging()
@@ -197,31 +196,31 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.rev_range and not args.no_gerrit:
- print >>sys.stderr, "--rev-range works only with --no-gerrit"
+ print("--rev-range works only with --no-gerrit", file=sys.stderr)
sys.exit(1)
# Find the gerrit revision URL, if applicable.
if not args.no_gerrit:
revision_url = get_gerrit_revision_url(args.rev)
- print revision_url
+ print(revision_url)
# Run clang-tidy and parse the output.
clang_output = run_tidy(args.rev, args.rev_range)
logging.info("Clang output")
logging.info(clang_output)
if args.no_gerrit:
- print >>sys.stderr, "Skipping gerrit"
+ print("Skipping gerrit", file=sys.stderr)
sys.exit(0)
logging.info("=" * 80)
parsed = parse_clang_output(clang_output)
if not parsed:
- print >>sys.stderr, "No warnings"
+ print("No warnings", file=sys.stderr)
sys.exit(0)
- print "Parsed clang warnings:"
- print json.dumps(parsed, indent=4)
+ print("Parsed clang warnings:")
+ print(json.dumps(parsed, indent=4))
# Post the output as comments to the gerrit URL.
gerrit_json_obj = create_gerrit_json_obj(parsed)
- print "Will post to gerrit:"
- print json.dumps(gerrit_json_obj, indent=4)
+ print("Will post to gerrit:")
+ print(json.dumps(gerrit_json_obj, indent=4))
post_comments(revision_url, gerrit_json_obj)
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index 8636314..9721460 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -388,7 +388,7 @@ def create_task_json(staging,
Alternatively, if 'retry_all_tests' is True, all tests will be retried.
"""
tasks = []
- with file(staging.archive_dump_path(), "r") as isolate_dump:
+ with open(staging.archive_dump_path(), "r") as isolate_dump:
inmap = json.load(isolate_dump)
# Some versions of 'isolate batcharchive' directly list the items in
@@ -415,7 +415,7 @@ def create_task_json(staging,
sys.exit(1)
outmap = {"tasks": tasks}
- with file(staging.tasks_json_path(), "wt") as f:
+ with open(staging.tasks_json_path(), "wt") as f:
json.dump(outmap, f)
@@ -655,7 +655,7 @@ def add_java_subparser(subparsers):
loop.set_defaults(func=loop_java_test)
def dump_base_deps(parser, options):
- print json.dumps(get_base_deps(create_dependency_extractor()))
+ print(json.dumps(get_base_deps(create_dependency_extractor())))
def add_internal_commands(subparsers):
p = subparsers.add_parser('internal', help="[Internal commands not for users]")
diff --git a/build-support/iwyu.py b/build-support/iwyu.py
index ff1a6ed..06efbbb 100755
--- a/build-support/iwyu.py
+++ b/build-support/iwyu.py
@@ -173,7 +173,7 @@ def _do_iwyu(flags, paths):
iwyu_output = _run_iwyu_tool(paths)
if flags.dump_iwyu_output:
logging.info("Dumping iwyu output to %s", flags.dump_iwyu_output)
- with file(flags.dump_iwyu_output, "w") as f:
+ with open(flags.dump_iwyu_output, "w") as f:
print(iwyu_output, file=f)
stream = BytesIO(iwyu_output)
fixer_flags = _get_fixer_flags(flags)
diff --git a/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py b/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py
index 87bb602..562c1f8 100755
--- a/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py
+++ b/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py
@@ -258,11 +258,11 @@ def prep_artifact_dirs(config):
"""
if not os.path.exists(config[ARTIFACT_ROOT]):
- os.makedirs(config[ARTIFACT_ROOT], mode=0755)
+ os.makedirs(config[ARTIFACT_ROOT], mode=0o755)
if not os.path.exists(config[ARTIFACT_BIN_DIR]):
- os.makedirs(config[ARTIFACT_BIN_DIR], mode=0755)
+ os.makedirs(config[ARTIFACT_BIN_DIR], mode=0o755)
if not os.path.exists(config[ARTIFACT_LIB_DIR]):
- os.makedirs(config[ARTIFACT_LIB_DIR], mode=0755)
+ os.makedirs(config[ARTIFACT_LIB_DIR], mode=0o755)
def copy_file(src, dest):
"""
diff --git a/build-support/parse_test_failure.py b/build-support/parse_test_failure.py
index dcb6cfc..82abe26 100755
--- a/build-support/parse_test_failure.py
+++ b/build-support/parse_test_failure.py
@@ -289,7 +289,8 @@ class Test(unittest.TestCase):
base, _ = os.path.splitext(child)
p = LogParser()
- p.parse_text(file(os.path.join(self._TEST_DIR, child)).read())
+ with open(os.path.join(self._TEST_DIR, child)) as f:
+ p.parse_text(f.read())
self._do_test(p.text_failure_summary(), base + "-out.txt")
self._do_test(p.xml_failure_summary(), base + "-out.xml")
@@ -297,10 +298,11 @@ class Test(unittest.TestCase):
path = os.path.join(self._TEST_DIR, filename)
if self.regenerate:
print("Regenerating %s" % path)
- with file(path, "w") as f:
+ with open(path, "w") as f:
f.write(got_value)
else:
- self.assertEquals(got_value, file(path).read())
+ with open(path) as f:
+ self.assertEquals(got_value, f.read())
def main():
@@ -312,10 +314,12 @@ def main():
args = parser.parse_args()
if args.path:
- in_file = file(args.path)
+ in_file = open(args.path)
else:
in_file = sys.stdin
log_text = in_file.read(MAX_MEMORY)
+ if in_file is not sys.stdin:
+ in_file.close()
format = args.xml and 'xml' or 'text'
sys.stdout.write(extract_failure_summary(log_text, format))
diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py
index f8d0ece..cc19f8f 100755
--- a/build-support/push_to_asf.py
+++ b/build-support/push_to_asf.py
@@ -196,7 +196,7 @@ def do_update(branch, gerrit_sha, apache_sha):
print(Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd))
subprocess.check_call(cmd)
print(Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET)
- print
+ print()
def main():
diff --git a/build-support/release/check-rat-report.py b/build-support/release/check-rat-report.py
index 213fd9b..4721452 100755
--- a/build-support/release/check-rat-report.py
+++ b/build-support/release/check-rat-report.py
@@ -53,5 +53,5 @@ for r in resources:
if not all_ok:
sys.exit(1)
-print 'OK'
+print('OK')
sys.exit(0)
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index 7bd7620..f9b12c7 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -50,7 +50,7 @@ def is_elf_binary(path):
if not os.path.isfile(path) or os.path.islink(path):
return False
try:
- with file(path, "rb") as f:
+ with open(path, "rb") as f:
magic = f.read(4)
return magic == "\x7fELF"
except:
@@ -186,7 +186,7 @@ def main():
if not os.path.exists(test_tmpdir):
os.makedirs(test_tmpdir)
cmd = [find_java()] + args
- stdout = stderr = file(os.path.join(test_logdir, "test-output.txt"), "w")
+ stdout = stderr = open(os.path.join(test_logdir, "test-output.txt"), "w")
else:
raise ValueError("invalid test language: " + options.test_language)
logging.info("Running command: ", cmd)
diff --git a/build-support/test_result_server.py b/build-support/test_result_server.py
index 6f037bf..15e859e 100755
--- a/build-support/test_result_server.py
+++ b/build-support/test_result_server.py
@@ -56,9 +56,13 @@ import logging
import MySQLdb
import os
import parse_test_failure
-from StringIO import StringIO
+from io import StringIO
import threading
import uuid
+try:
+ xrange # For Python 2
+except NameError:
+ xrange = range # For Python 3
def percent_rate(num, denom):
if denom == 0:
diff --git a/examples/python/graphite-kudu/kudu/kudu_graphite.py b/examples/python/graphite-kudu/kudu/kudu_graphite.py
index 1765303..4a7d347 100644
--- a/examples/python/graphite-kudu/kudu/kudu_graphite.py
+++ b/examples/python/graphite-kudu/kudu/kudu_graphite.py
@@ -299,6 +299,6 @@ class KuduFinder(object):
try:
for node in self._find_nodes_from_pattern(self.kudu_table, query.pattern):
yield node
- except Exception, e:
+ except Exception as e:
log.exception(e)
raise
diff --git a/src/kudu/benchmarks/wal_hiccup-parser.py b/src/kudu/benchmarks/wal_hiccup-parser.py
index 7ba3d5f..9da8f42 100755
--- a/src/kudu/benchmarks/wal_hiccup-parser.py
+++ b/src/kudu/benchmarks/wal_hiccup-parser.py
@@ -32,7 +32,7 @@ import sys
def main():
if len(sys.argv) != 2:
- print "Usage: %s <log output>" % (sys.argv[0],)
+ print("Usage: %s <log output>" % (sys.argv[0],))
return
cols = list()
cols_printed = False
@@ -48,9 +48,9 @@ def main():
# End of a test result.
elif "-------" in line and vals is not None:
if not cols_printed:
- print ",".join(cols)
+ print(",".join(cols))
cols_printed = True
- print ",".join(vals)
+ print(",".join(vals))
vals = None
# Entry in a test result.
diff --git a/src/kudu/experiments/merge-test.py b/src/kudu/experiments/merge-test.py
index feddee3..419301c 100755
--- a/src/kudu/experiments/merge-test.py
+++ b/src/kudu/experiments/merge-test.py
@@ -27,6 +27,10 @@ import logging
import random
import time
import unittest
+try:
+ xrange # For Python 2
+except NameError:
+ xrange = range # For Python 3
# Adjustable experiment parameters.
BLOCK_SIZE = 1000
diff --git a/src/kudu/scripts/get-job-stats-from-mysql.py b/src/kudu/scripts/get-job-stats-from-mysql.py
index 5748886..2fd062f 100644
--- a/src/kudu/scripts/get-job-stats-from-mysql.py
+++ b/src/kudu/scripts/get-job-stats-from-mysql.py
@@ -52,7 +52,7 @@ with con:
days = sys.argv[2]
cur.execute("select workload, runtime, build_number from kudu_perf_tpch where workload like %s AND curr_date >= DATE_SUB(NOW(), INTERVAL %s DAY) and runtime != 0 ORDER BY workload, build_number, curr_date", (workload, days))
rows = cur.fetchall()
- print 'workload', '\t', 'runtime', '\t', 'build_number'
+ print('workload', '\t', 'runtime', '\t', 'build_number')
for row in rows:
- print row[0], '\t', row[1], '\t', row[2]
+ print(row[0], '\t', row[1], '\t', row[2])
diff --git a/src/kudu/scripts/graph-metrics.py b/src/kudu/scripts/graph-metrics.py
index 2ed126c..853e2e4 100755
--- a/src/kudu/scripts/graph-metrics.py
+++ b/src/kudu/scripts/graph-metrics.py
@@ -69,9 +69,9 @@ def main():
keys = get_keys(data)
with sys.stdout as f:
- print >>f, "\t".join(keys)
+ print("\t".join(keys), file=f)
for row in data:
- print >>f, "\t".join([str(row.get(k, 0)) for k in keys])
+ print("\t".join([str(row.get(k, 0)) for k in keys]), file=f)
if __name__ == "__main__":
diff --git a/src/kudu/scripts/max_skew_estimate.py b/src/kudu/scripts/max_skew_estimate.py
index 841c75a..9dd4732 100755
--- a/src/kudu/scripts/max_skew_estimate.py
+++ b/src/kudu/scripts/max_skew_estimate.py
@@ -23,6 +23,10 @@
import math
import random
import sys
+try:
+ xrange # For Python 2
+except NameError:
+ xrange = range # For Python 3
# Replicates Random::ReservoirSample from kudu/util/random.h.
def reservoir_sample(n, sample_size, avoid):
@@ -74,13 +78,13 @@ def generate_max_skew(num_servers, num_tablets, rf):
def main():
args = sys.argv
if len(args) != 5:
- print "max_skew_estimate.py <num trials> <num servers> <num_tablets> <repl factor>"
+ print("max_skew_estimate.py <num trials> <num servers> <num_tablets> <repl factor>")
sys.exit(1)
num_trials, num_servers, num_tablets, rf = int(args[1]), int(args[2]), int(args[3]), int(args[4])
skews = [generate_max_skew(num_servers, num_tablets, rf) for _ in xrange(num_trials)]
skews.sort()
for p in [5, 25, 50, 75, 99]:
- print "%02d percentile: %d" % (p, percentile(skews, p))
+ print("{:02d} percentile: {:d}".format(p, percentile(skews, p)))
if __name__ == "__main__":
main()
diff --git a/src/kudu/scripts/parse_metrics_log.py b/src/kudu/scripts/parse_metrics_log.py
index f618010..af8c8c8 100644
--- a/src/kudu/scripts/parse_metrics_log.py
+++ b/src/kudu/scripts/parse_metrics_log.py
@@ -237,9 +237,9 @@ def process(aggregated_prev, aggregated_cur):
stats = histogram_stats(aggregated_prev, aggregated_cur, metric)
calc_vals.extend([stats['p50'], stats['p95'], stats['p99'], stats['p999'], stats['max']])
- print (aggregated_cur['ts'] + aggregated_prev['ts'])/2, \
- cache_ratio, \
- " ".join(str(x) for x in calc_vals)
+ print((aggregated_cur['ts'] + aggregated_prev['ts'])/2,
+ cache_ratio,
+ " ".join(str(x) for x in calc_vals))
return aggregated_cur
def main(argv):
@@ -254,13 +254,13 @@ def main(argv):
simple_headers.append(header + "_p999")
simple_headers.append(header + "_max")
- print "time cache_hit_ratio", " ".join(simple_headers)
+ print("time cache_hit_ratio", " ".join(simple_headers))
for path in sorted(argv[1:]):
if path.endswith(".gz"):
f = gzip.GzipFile(path)
else:
- f = file(path)
+ f = open(path)
for line_number, line in enumerate(f, start=1):
# Only parse out the "metrics" lines.
try:
diff --git a/src/kudu/scripts/write-jobs-stats-to-mysql.py b/src/kudu/scripts/write-jobs-stats-to-mysql.py
index a6b092a..0493a67 100644
--- a/src/kudu/scripts/write-jobs-stats-to-mysql.py
+++ b/src/kudu/scripts/write-jobs-stats-to-mysql.py
@@ -30,7 +30,7 @@ pwd = os.environ["MYSQLPWD"]
db = os.environ["MYSQLDB"]
con = mdb.connect(host, user, pwd, db)
-print "Connected to mysql"
+print("Connected to mysql")
with con:
cur = con.cursor()
job_name = sys.argv[1]