You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2022/10/28 11:01:55 UTC
[hbase] branch branch-2.4 updated: HBASE-27450 Update all our python scripts to use python3 (#4851)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 73169be87cb HBASE-27450 Update all our python scripts to use python3 (#4851)
73169be87cb is described below
commit 73169be87cb89c32c89316b77eced88db959a866
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Fri Oct 28 18:41:47 2022 +0800
HBASE-27450 Update all our python scripts to use python3 (#4851)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
(cherry picked from commit cdabfd3ca811d1c5ef79e9b6bd8106b2d6d12cd5)
---
dev-support/Dockerfile | 10 +-
dev-support/Jenkinsfile | 12 +-
dev-support/Jenkinsfile_GitHub | 8 +-
dev-support/checkcompatibility.py | 54 ++--
dev-support/checkstyle_report.py | 56 ++--
dev-support/flaky-tests/findHangingTests.py | 38 +--
.../flaky-tests/flaky-reporting.Jenkinsfile | 2 +-
dev-support/flaky-tests/report-flakies.py | 28 +-
dev-support/gen_redirect_html.py | 8 +-
dev-support/submit-patch.py | 311 ---------------------
10 files changed, 112 insertions(+), 415 deletions(-)
diff --git a/dev-support/Dockerfile b/dev-support/Dockerfile
index 923e26563b3..44822e9fb44 100644
--- a/dev-support/Dockerfile
+++ b/dev-support/Dockerfile
@@ -20,16 +20,14 @@
#
# Specifically, it's used for the flaky test reporting job defined in
# dev-support/flaky-tests/flaky-reporting.Jenkinsfile
-FROM ubuntu:18.04
+FROM ubuntu:22.04
COPY . /hbase/dev-support
RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
&& DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
- curl='7.58.0-*' \
- python2.7='2.7.17-*' \
- python-pip='9.0.1-*' \
- python-setuptools='39.0.1-*' \
+ curl='7.81.0-*' \
+ python3-pip='22.0.2+dfsg-*' \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-RUN pip install -r /hbase/dev-support/python-requirements.txt
+RUN pip3 install -r /hbase/dev-support/python-requirements.txt
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index f287f01f637..2098fff8008 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -267,14 +267,14 @@ pipeline {
if [ -d "${OUTPUT_DIR}/branch-site" ]; then
echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
rm -rf "${OUTPUT_DIR}/branch-site"
- python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
else
echo "No branch-site, skipping"
fi
if [ -d "${OUTPUT_DIR}/patch-site" ]; then
echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
rm -rf "${OUTPUT_DIR}/patch-site"
- python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
else
echo "No patch-site, skipping"
fi
@@ -384,7 +384,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -493,7 +493,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -604,7 +604,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -773,7 +773,7 @@ pipeline {
if [ -f "${SRC_TAR}" ]; then
echo "Remove ${SRC_TAR} for saving space"
rm -rf "${SRC_TAR}"
- python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
else
echo "No hbase-src.tar.gz, skipping"
fi
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index 4eee610b8b1..351f1f30b54 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -143,14 +143,14 @@ pipeline {
if [ -d "${PATCHDIR}/branch-site" ]; then
echo "Remove ${PATCHDIR}/branch-site for saving space"
rm -rf "${PATCHDIR}/branch-site"
- python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
+ python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
else
echo "No branch-site, skipping"
fi
if [ -d "${PATCHDIR}/patch-site" ]; then
echo "Remove ${PATCHDIR}/patch-site for saving space"
rm -rf "${PATCHDIR}/patch-site"
- python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
+ python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
else
echo "No patch-site, skipping"
fi
@@ -282,7 +282,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip"
- python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+ python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -415,7 +415,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip"
- python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+ python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
diff --git a/dev-support/checkcompatibility.py b/dev-support/checkcompatibility.py
index 4cfdc4afeae..2f324856134 100755
--- a/dev-support/checkcompatibility.py
+++ b/dev-support/checkcompatibility.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -32,7 +32,7 @@
# --annotation org.apache.yetus.audience.InterfaceAudience.LimitedPrivate \
# --include-file "hbase-*" \
# --known_problems_path ~/known_problems.json \
-# rel/1.0.0 branch-1.2
+# rel/1.3.0 branch-1.4
import json
import logging
@@ -41,7 +41,9 @@ import re
import shutil
import subprocess
import sys
-import urllib2
+import urllib.request
+import urllib.error
+import urllib.parse
from collections import namedtuple
try:
import argparse
@@ -55,11 +57,11 @@ REPO_DIR = os.getcwd()
def check_output(*popenargs, **kwargs):
- """ Run command with arguments and return its output as a byte string.
- Backported from Python 2.7 as it's implemented as pure python on stdlib.
- >>> check_output(['/usr/bin/python', '--version'])
- Python 2.6.2 """
- process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+ """ Run command with arguments and return its output as a byte string. """
+ process = subprocess.Popen(stdout=subprocess.PIPE,
+ universal_newlines=True,
+ *popenargs,
+ **kwargs)
output, _ = process.communicate()
retcode = process.poll()
if retcode:
@@ -69,7 +71,7 @@ def check_output(*popenargs, **kwargs):
error = subprocess.CalledProcessError(retcode, cmd)
error.output = output
raise error
- return output
+ return output.strip()
def get_repo_dir():
@@ -159,7 +161,7 @@ def checkout_java_acc(force):
url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz"
scratch_dir = get_scratch_dir()
path = os.path.join(scratch_dir, os.path.basename(url))
- jacc = urllib2.urlopen(url)
+ jacc = urllib.request.urlopen(url)
with open(path, 'wb') as w:
w.write(jacc.read())
@@ -194,8 +196,8 @@ def ascii_encode_dict(data):
""" Iterate through a dictionary of data and convert all unicode to ascii.
This method was taken from
stackoverflow.com/questions/9590382/forcing-python-json-module-to-work-with-ascii """
- ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
- return dict(map(ascii_encode, pair) for pair in data.items())
+ ascii_encode = lambda x: x.encode('ascii') if isinstance(x, str) else x
+ return dict(list(map(ascii_encode, pair)) for pair in list(data.items()))
def process_json(path):
@@ -227,9 +229,9 @@ def compare_results(tool_results, known_issues, compare_warnings):
unexpected_issues = [unexpected_issue(check=check, issue_type=issue_type,
known_count=known_count,
observed_count=tool_results[check][issue_type])
- for check, known_issue_counts in known_issues.items()
- for issue_type, known_count in known_issue_counts.items()
- if tool_results[check][issue_type] > known_count]
+ for check, known_issue_counts in list(known_issues.items())
+ for issue_type, known_count in list(known_issue_counts.items())
+ if compare_tool_results_count(tool_results, check, issue_type, known_count)]
if not compare_warnings:
unexpected_issues = [tup for tup in unexpected_issues
@@ -241,6 +243,14 @@ def compare_results(tool_results, known_issues, compare_warnings):
return bool(unexpected_issues)
+def compare_tool_results_count(tool_results, check, issue_type, known_count):
+ """ Check problem counts are no more than the known count.
+ (This function exists just so can add in logging; previous was inlined
+ one-liner but this made it hard debugging)
+ """
+ # logging.info("known_count=%s, check key=%s, tool_results=%s, issue_type=%s",
+ # str(known_count), str(check), str(tool_results), str(issue_type))
+ return tool_results[check][issue_type] > known_count
def process_java_acc_output(output):
""" Process the output string to find the problems and warnings in both the
@@ -299,14 +309,14 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annot
logging.info("Annotations are: %s", annotations)
annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
logging.info("Annotations path: %s", annotations_path)
- with file(annotations_path, "w") as f:
+ with open(annotations_path, "w") as f:
f.write('\n'.join(annotations))
args.extend(["-annotations-list", annotations_path])
if skip_annotations is not None:
skip_annotations_path = os.path.join(
get_scratch_dir(), "skip_annotations.txt")
- with file(skip_annotations_path, "w") as f:
+ with open(skip_annotations_path, "w") as f:
f.write('\n'.join(skip_annotations))
args.extend(["-skip-annotations-list", skip_annotations_path])
@@ -325,14 +335,14 @@ def get_known_problems(json_path, src_rev, dst_rev):
keys in the format source_branch/destination_branch and the values
dictionaries with binary and source problems and warnings
Example:
- {'branch-1.0.0': {
- 'rel/1.0.0': {'binary': {'problems': 123, 'warnings': 16},
+ {'branch-1.3': {
+ 'rel/1.3.0': {'binary': {'problems': 123, 'warnings': 16},
'source': {'problems': 167, 'warnings': 1}},
- 'branch-1.2.0': {'binary': {'problems': 0, 'warnings': 0},
+ 'branch-1.4': {'binary': {'problems': 0, 'warnings': 0},
'source': {'problems': 0, 'warnings': 0}}
},
- 'branch-1.2.0': {
- 'rel/1.2.1': {'binary': {'problems': 13, 'warnings': 1},
+ 'branch-1.4': {
+ 'rel/1.4.1': {'binary': {'problems': 13, 'warnings': 1},
'source': {'problems': 23, 'warnings': 0}}
}
} """
diff --git a/dev-support/checkstyle_report.py b/dev-support/checkstyle_report.py
index c5810be152b..99092c23b8c 100755
--- a/dev-support/checkstyle_report.py
+++ b/dev-support/checkstyle_report.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -29,8 +29,8 @@ import xml.etree.ElementTree as etree
from collections import defaultdict
if len(sys.argv) != 3 :
- print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
- exit(1)
+ print("usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0])
+ sys.exit(1)
def path_key(x):
path = x.attrib['name']
@@ -40,8 +40,8 @@ def error_name(x):
error_class = x.attrib['source']
return error_class[error_class.rfind(".") + 1:]
-def print_row(path, error, master_errors, patch_errors):
- print '%s\t%s\t%s\t%s' % (path,error, master_errors,patch_errors)
+def print_row(path, err, master_errors, patch_errors):
+ print('%s\t%s\t%s\t%s' % (path, err, master_errors, patch_errors))
master = etree.parse(sys.argv[1])
patch = etree.parse(sys.argv[2])
@@ -49,32 +49,32 @@ patch = etree.parse(sys.argv[2])
master_dict = defaultdict(int)
ret_value = 0
-for child in master.getroot().getchildren():
+for child in list(master.getroot()):
if child.tag != 'file':
- continue
+ continue
file = path_key(child)
- for error_tag in child.getchildren():
- error = error_name(error_tag)
- if (file, error) in master_dict:
- master_dict[(file, error)] += 1
- else:
- master_dict[(file, error)] = 1
+ for error_tag in list(child):
+ error = error_name(error_tag)
+ if (file, error) in master_dict:
+ master_dict[(file, error)] += 1
+ else:
+ master_dict[(file, error)] = 1
-for child in patch.getroot().getchildren():
- if child.tag != 'file':
- continue
- temp_dict = defaultdict(int)
- for error_tag in child.getchildren():
- error = error_name(error_tag)
- if error in temp_dict:
- temp_dict[error] += 1
- else:
- temp_dict[error] = 1
+for child in list(patch.getroot()):
+ if child.tag != 'file':
+ continue
+ temp_dict = defaultdict(int)
+ for error_tag in list(child):
+ error = error_name(error_tag)
+ if error in temp_dict:
+ temp_dict[error] += 1
+ else:
+ temp_dict[error] = 1
- file = path_key(child)
- for error, count in temp_dict.iteritems():
- if count > master_dict[(file, error)]:
- print_row(file, error, master_dict[(file, error)], count)
- ret_value = 1
+ file = path_key(child)
+ for error, count in temp_dict.items():
+ if count > master_dict[(file, error)]:
+ print_row(file, error, master_dict[(file, error)], count)
+ ret_value = 1
sys.exit(ret_value)
diff --git a/dev-support/flaky-tests/findHangingTests.py b/dev-support/flaky-tests/findHangingTests.py
index f8416e0a1e1..bb0a9a5be61 100755
--- a/dev-support/flaky-tests/findHangingTests.py
+++ b/dev-support/flaky-tests/findHangingTests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -45,8 +45,8 @@ def get_bad_tests(console_url):
"""
response = requests.get(console_url)
if response.status_code != 200:
- print "Error getting consoleText. Response = {} {}".format(
- response.status_code, response.reason)
+ print("Error getting consoleText. Response = {} {}".format(
+ response.status_code, response.reason))
return
# All tests: All testcases which were run.
@@ -59,13 +59,13 @@ def get_bad_tests(console_url):
hanging_tests_set = set()
failed_tests_set = set()
timeout_tests_set = set()
- for line in response.content.splitlines():
+ for line in response.content.decode("utf-8").splitlines():
result1 = re.findall("Running org.apache.hadoop.hbase.(.*)", line)
if len(result1) == 1:
test_case = result1[0]
if test_case in all_tests_set:
- print ("ERROR! Multiple tests with same name '{}'. Might get wrong results "
- "for this test.".format(test_case))
+ print(("ERROR! Multiple tests with same name '{}'. Might get wrong results "
+ "for this test.".format(test_case)))
else:
hanging_tests_set.add(test_case)
all_tests_set.add(test_case)
@@ -75,9 +75,9 @@ def get_bad_tests(console_url):
if "FAILURE!" in line:
failed_tests_set.add(test_case)
if test_case not in hanging_tests_set:
- print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results "
+ print(("ERROR! No test '{}' found in hanging_tests. Might get wrong results "
"for this test. This may also happen if maven is set to retry failing "
- "tests.".format(test_case))
+ "tests.".format(test_case)))
else:
hanging_tests_set.remove(test_case)
result3 = re.match("^\\s+(\\w*).*\\sTestTimedOut", line)
@@ -86,30 +86,30 @@ def get_bad_tests(console_url):
timeout_tests_set.add(test_case)
for bad_string in BAD_RUN_STRINGS:
if re.match(".*" + bad_string + ".*", line):
- print "Bad string found in build:\n > {}".format(line)
- print "Result > total tests: {:4} failed : {:4} timedout : {:4} hanging : {:4}".format(
- len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set))
+ print("Bad string found in build:\n > {}".format(line))
+ print("Result > total tests: {:4} failed : {:4} timedout : {:4} hanging : {:4}".format(
+ len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set)))
return [all_tests_set, failed_tests_set, timeout_tests_set, hanging_tests_set]
if __name__ == "__main__":
if len(sys.argv) != 2:
- print "ERROR : Provide the jenkins job console URL as the only argument."
+ print("ERROR : Provide the jenkins job console URL as the only argument.")
sys.exit(1)
- print "Fetching {}".format(sys.argv[1])
+ print("Fetching {}".format(sys.argv[1]))
result = get_bad_tests(sys.argv[1])
if not result:
sys.exit(1)
[all_tests, failed_tests, timedout_tests, hanging_tests] = result
- print "Found {} hanging tests:".format(len(hanging_tests))
+ print("Found {} hanging tests:".format(len(hanging_tests)))
for test in hanging_tests:
- print test
- print "\n"
- print "Found {} failed tests of which {} timed out:".format(
- len(failed_tests), len(timedout_tests))
+ print(test)
+ print("\n")
+ print("Found {} failed tests of which {} timed out:".format(
+ len(failed_tests), len(timedout_tests)))
for test in failed_tests:
- print "{0} {1}".format(test, ("(Timed Out)" if test in timedout_tests else ""))
+ print("{0} {1}".format(test, ("(Timed Out)" if test in timedout_tests else "")))
print ("\nA test may have had 0 or more atomic test failures before it timed out. So a "
"'Timed Out' test may have other errors too.")
diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
index b37045b3a6c..f5090095453 100644
--- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
+++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
@@ -47,7 +47,7 @@ pipeline {
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 50)
docker build -t hbase-dev-support dev-support
docker run --ulimit nproc=12500 -v "${WORKSPACE}":/hbase -u `id -u`:`id -g` --workdir=/hbase hbase-dev-support \
- python dev-support/flaky-tests/report-flakies.py --mvn -v -o output "${flaky_args[@]}"
+ ./dev-support/flaky-tests/report-flakies.py --mvn -v -o output "${flaky_args[@]}"
'''
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
diff --git a/dev-support/flaky-tests/report-flakies.py b/dev-support/flaky-tests/report-flakies.py
index e2d30b0d6dc..16096e3344a 100755
--- a/dev-support/flaky-tests/report-flakies.py
+++ b/dev-support/flaky-tests/report-flakies.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -140,7 +140,7 @@ def expand_multi_config_projects(cli_args):
raise Exception("Failed to get job information from jenkins for url '" + job_url +
"'. Jenkins returned HTTP status " + str(request.status_code))
response = request.json()
- if response.has_key("activeConfigurations"):
+ if "activeConfigurations" in response:
for config in response["activeConfigurations"]:
final_expanded_urls.append({'url':config["url"], 'max_builds': max_builds,
'excludes': excluded_builds, 'is_yetus': is_yetus})
@@ -167,7 +167,7 @@ for url_max_build in expanded_urls:
url = url_max_build["url"]
excludes = url_max_build["excludes"]
json_response = requests.get(url + "/api/json?tree=id,builds%5Bnumber,url%5D").json()
- if json_response.has_key("builds"):
+ if "builds" in json_response:
builds = json_response["builds"]
logger.info("Analyzing job: %s", url)
else:
@@ -238,27 +238,27 @@ for url_max_build in expanded_urls:
# Sort tests in descending order by flakyness.
sorted_test_to_build_ids = OrderedDict(
- sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True))
+ sorted(iter(test_to_build_ids.items()), key=lambda x: x[1]['flakyness'], reverse=True))
url_to_bad_test_results[url] = sorted_test_to_build_ids
if len(sorted_test_to_build_ids) > 0:
- print "URL: {}".format(url)
- print "{:>60} {:10} {:25} {}".format(
- "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness")
+ print("URL: {}".format(url))
+ print("{:>60} {:10} {:25} {}".format(
+ "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness"))
for bad_test in sorted_test_to_build_ids:
test_status = sorted_test_to_build_ids[bad_test]
- print "{:>60} {:10} {:7} ( {:4} / {:5} / {:5} ) {:2.0f}%".format(
+ print("{:>60} {:10} {:7} ( {:4} / {:5} / {:5} ) {:2.0f}%".format(
bad_test, len(test_status['all']), test_status['bad_count'],
len(test_status['failed']), len(test_status['timeout']),
- len(test_status['hanging']), test_status['flakyness'])
+ len(test_status['hanging']), test_status['flakyness']))
else:
- print "No flaky tests founds."
+ print("No flaky tests founds.")
if len(url_to_build_ids[url]) == len(build_ids_without_tests_run):
- print "None of the analyzed builds have test result."
+ print("None of the analyzed builds have test result.")
- print "Builds analyzed: {}".format(url_to_build_ids[url])
- print "Builds without any test runs: {}".format(build_ids_without_tests_run)
- print ""
+ print("Builds analyzed: {}".format(url_to_build_ids[url]))
+ print("Builds without any test runs: {}".format(build_ids_without_tests_run))
+ print("")
all_bad_tests = all_hanging_tests.union(all_failed_tests)
diff --git a/dev-support/gen_redirect_html.py b/dev-support/gen_redirect_html.py
index 8a24a81ac65..2689fd8aa4a 100755
--- a/dev-support/gen_redirect_html.py
+++ b/dev-support/gen_redirect_html.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -20,8 +20,8 @@ import sys
from string import Template
if len(sys.argv) != 2 :
- print "usage: %s <redirect url>" % sys.argv[0]
- exit(1)
+ print("usage: %s <redirect url>" % sys.argv[0])
+ sys.exit(1)
url = sys.argv[1].replace(" ", "%20")
template = Template("""<html>
@@ -34,4 +34,4 @@ template = Template("""<html>
</html>""")
output = template.substitute(url = url)
-print output
+print(output)
diff --git a/dev-support/submit-patch.py b/dev-support/submit-patch.py
deleted file mode 100755
index 8d13e20c84d..00000000000
--- a/dev-support/submit-patch.py
+++ /dev/null
@@ -1,311 +0,0 @@
-#!/usr/bin/env python2
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Makes a patch for the current branch, creates/updates the review board request and uploads new
-# patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming
-# rules. If no jira is specified, patch will be named (branch name).patch and jira and review board
-# are not updated. Review board id is retrieved from the remote link in the jira.
-# Print help: submit-patch.py --h
-import argparse
-from builtins import input, str
-import getpass
-import git
-import json
-import logging
-import os
-import re
-import requests
-import subprocess
-import sys
-
-parser = argparse.ArgumentParser(
- epilog = "To avoid having to enter jira/review board username/password every time, setup an "
- "encrypted ~/.apache-cred files as follows:\n"
- "1) Create a file with following single "
- "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", "
- "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n"
- "2) Encrypt it with openssl.\n"
- "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n"
- "3) Delete original file.\n"
- "Now onwards, you'll need to enter this encryption key only once per run. If you "
- "forget the key, simply regenerate ~/.apache-cred file again.",
- formatter_class=argparse.RawTextHelpFormatter
-)
-parser.add_argument("-b", "--branch",
- help = "Branch to use for generating diff. If not specified, tracking branch "
- "is used. If there is no tracking branch, error will be thrown.")
-
-# Arguments related to Jira.
-parser.add_argument("-jid", "--jira-id",
- help = "Jira id of the issue. If set, we deduce next patch version from "
- "attachments in the jira and also upload the new patch. Script will "
- "ask for jira username/password for authentication. If not set, "
- "patch is named <branch>.patch.")
-
-# Arguments related to Review Board.
-parser.add_argument("-srb", "--skip-review-board",
- help = "Don't create/update the review board.",
- default = False, action = "store_true")
-parser.add_argument("--reviewers",
- help = "Comma separated list of users to add as reviewers.")
-
-# Misc arguments
-parser.add_argument("--patch-dir", default = "~/patches",
- help = "Directory to store patch files. If it doesn't exist, it will be "
- "created. Default: ~/patches")
-parser.add_argument("--rb-repo", default = "hbase-git",
- help = "Review board repository. Default: hbase-git")
-args = parser.parse_args()
-
-# Setup logger
-logging.basicConfig()
-logger = logging.getLogger("submit-patch")
-logger.setLevel(logging.INFO)
-
-
-def log_fatal_and_exit(*arg):
- logger.fatal(*arg)
- sys.exit(1)
-
-
-def assert_status_code(response, expected_status_code, description):
- if response.status_code != expected_status_code:
- log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..",
- description, response.status_code, response.reason)
-
-
-# Make repo instance to interact with git repo.
-try:
- repo = git.Repo(os.getcwd())
- git = repo.git
-except git.exc.InvalidGitRepositoryError as e:
- log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of "
- "HBase's git repo.", e)
-
-logger.info(" Active branch: %s", repo.active_branch.name)
-# Do not proceed if there are uncommitted changes.
-if repo.is_dirty():
- log_fatal_and_exit(" Git status is dirty. Commit locally first.")
-
-
-# Returns base branch for creating diff.
-def get_base_branch():
- # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch.
- if args.branch is not None:
- base_branch = args.branch
- # Check that given branch exists.
- for ref in repo.refs:
- if ref.name == base_branch:
- return base_branch
- log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch)
- else:
- # if --branch is not set, use tracking branch as base branch for computing diff.
- # If there is no tracking branch, log error and quit.
- tracking_branch = repo.active_branch.tracking_branch()
- if tracking_branch is None:
- log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base "
- " branch for computing diff using --branch flag.")
- logger.info(" Using tracking branch as base branch")
- return tracking_branch.name
-
-
-# Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is
-# specified, patch is name (branch name).patch.
-def get_patch_name(branch):
- if args.jira_id is None:
- return branch + ".patch"
-
- patch_name_prefix = args.jira_id.upper() + "." + branch
- return get_patch_name_with_version(patch_name_prefix)
-
-
-# Fetches list of attachments from the jira, deduces next version for the patch and returns final
-# patch name.
-def get_patch_name_with_version(patch_name_prefix):
- # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637.
- # Using crude way to get list of attachments.
- url = "https://issues.apache.org/jira/browse/" + args.jira_id
- logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url)
- html = requests.get(url)
- if html.status_code == 404:
- log_fatal_and_exit(" Invalid jira id : %s", args.jira_id)
- if html.status_code != 200:
- log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code)
- # Iterate over patch names starting from version 1 and return when name is not already used.
- content = str(html.content, 'utf-8')
- for i in range(1, 1000):
- name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch"
- if name not in content:
- return name
-
-
-# Validates that patch directory exists, if not, creates it.
-def validate_patch_dir(patch_dir):
- # Create patch_dir if it doesn't exist.
- if not os.path.exists(patch_dir):
- logger.warn(" Patch directory doesn't exist. Creating it.")
- os.mkdir(patch_dir)
- else:
- # If patch_dir exists, make sure it's a directory.
- if not os.path.isdir(patch_dir):
- log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.",
- patch_dir)
-
-
-# Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if
-# - base_branch has commits not in current branch
-# - current branch is same as base branch
-# - current branch is ahead of base_branch by more than 1 commits
-def check_diff_between_branches(base_branch):
- only_in_base_branch = list(repo.iter_commits("HEAD.." + base_branch))
- only_in_active_branch = list(repo.iter_commits(base_branch + "..HEAD"))
- if len(only_in_base_branch) != 0:
- log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase "
- "and try again.", base_branch, len(only_in_base_branch))
- if len(only_in_active_branch) == 0:
- log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch)
- if len(only_in_active_branch) > 1:
- log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single "
- "commit and try again.", base_branch, len(only_in_active_branch))
-
-
-# If ~/.apache-creds is present, load credentials from it otherwise prompt user.
-def get_credentials():
- creds = dict()
- creds_filepath = os.path.expanduser("~/.apache-creds")
- if os.path.exists(creds_filepath):
- try:
- logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials")
- content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath,
- shell=True)
- except subprocess.CalledProcessError as e:
- log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..")
- creds = json.loads(content)
- else:
- creds['jira_username'] = input("Jira username:")
- creds['jira_password'] = getpass.getpass("Jira password:")
- if not args.skip_review_board:
- creds['rb_username'] = input("Review Board username:")
- creds['rb_password'] = getpass.getpass("Review Board password:")
- return creds
-
-
-def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds):
- # Upload patch to jira using REST API.
- headers = {'X-Atlassian-Token': 'no-check'}
- files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')}
- jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
- attachment_url = issue_url + "/attachments"
- r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth)
- assert_status_code(r, 200, "uploading patch to jira")
-
-
-def get_jira_summary(issue_url):
- r = requests.get(issue_url + "?fields=summary")
- assert_status_code(r, 200, "fetching jira summary")
- return json.loads(r.content)["fields"]["summary"]
-
-
-def get_review_board_id_if_present(issue_url, rb_link_title):
- r = requests.get(issue_url + "/remotelink")
- assert_status_code(r, 200, "fetching remote links")
- links = json.loads(r.content)
- for link in links:
- if link["object"]["title"] == rb_link_title:
- res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"])
- return res.group(1)
- return None
-
-
-base_branch = get_base_branch()
-# Remove remote repo name from branch name if present. This assumes that we don't use '/' in
-# actual branch names.
-base_branch_without_remote = base_branch.split('/')[-1]
-logger.info(" Base branch: %s", base_branch)
-
-check_diff_between_branches(base_branch)
-
-patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir))
-logger.info(" Patch directory: %s", patch_dir)
-validate_patch_dir(patch_dir)
-
-patch_filename = get_patch_name(base_branch_without_remote)
-logger.info(" Patch name: %s", patch_filename)
-patch_filepath = os.path.join(patch_dir, patch_filename)
-
-diff = git.format_patch(base_branch, stdout = True)
-with open(patch_filepath, "w") as f:
- f.write(diff.encode('utf8'))
-
-if args.jira_id is not None:
- creds = get_credentials()
- issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id
-
- attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds)
-
- if not args.skip_review_board:
- rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password'])
-
- rb_link_title = "Review Board (" + base_branch_without_remote + ")"
- rb_id = get_review_board_id_if_present(issue_url, rb_link_title)
-
- # If no review board link found, create new review request and add its link to jira.
- if rb_id is None:
- reviews_url = "https://reviews.apache.org/api/review-requests/"
- data = {"repository" : "hbase-git"}
- r = requests.post(reviews_url, data = data, auth = rb_auth)
- assert_status_code(r, 201, "creating new review request")
- review_request = json.loads(r.content)["review_request"]
- absolute_url = review_request["absolute_url"]
- logger.info(" Created new review request: %s", absolute_url)
-
- # Use jira summary as review's summary too.
- summary = get_jira_summary(issue_url)
- # Use commit message as description.
- description = repo.head.commit.message
- update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase",
- "target_people" : args.reviewers, "summary" : summary,
- "description" : description }
- draft_url = review_request["links"]["draft"]["href"]
- r = requests.put(draft_url, data = update_draft_data, auth = rb_auth)
- assert_status_code(r, 200, "updating review draft")
-
- draft_request = json.loads(r.content)["draft"]
- diff_url = draft_request["links"]["draft_diffs"]["href"]
- files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
- r = requests.post(diff_url, files = files, auth = rb_auth)
- assert_status_code(r, 201, "uploading diff to review draft")
-
- r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
- assert_status_code(r, 200, "publishing review request")
-
- # Add link to review board in the jira.
- remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}})
- jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
- r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth,
- headers={'Content-Type':'application/json'})
- else:
- logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id)
- draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/"
- diff_url = draft_url + "diffs/"
- files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
- r = requests.post(diff_url, files = files, auth = rb_auth)
- assert_status_code(r, 201, "uploading diff to review draft")
-
- r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
- assert_status_code(r, 200, "publishing review request")