You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/04/20 05:54:16 UTC
[3/3] kudu git commit: build-support: add a nicer logger to python
scripts
build-support: add a nicer logger to python scripts
Change-Id: Ic648fadfbb11eb3e0a8f137ae3f3143e52149101
Reviewed-on: http://gerrit.cloudera.org:8080/10131
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/953a7070
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/953a7070
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/953a7070
Branch: refs/heads/master
Commit: 953a7070f6a743410f9b8dc74e85b66023d8ab56
Parents: b463969
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Apr 19 16:44:22 2018 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Fri Apr 20 05:52:05 2018 +0000
----------------------------------------------------------------------
build-support/build_source_release.py | 5 +++--
build-support/check_compatibility.py | 4 ++--
build-support/clang_tidy_gerrit.py | 4 +++-
build-support/dist_test.py | 24 +++++++++++++-----------
build-support/gen_version_info.py | 5 +++--
build-support/iwyu.py | 5 ++---
build-support/kudu_util.py | 14 ++++++++++++++
build-support/push_to_asf.py | 4 ++--
8 files changed, 42 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/build_source_release.py
----------------------------------------------------------------------
diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py
index 35ffa40..5ef39b2 100755
--- a/build-support/build_source_release.py
+++ b/build-support/build_source_release.py
@@ -31,7 +31,8 @@ try:
except ImportError:
import urllib
-from kudu_util import check_output, confirm_prompt, Colors, get_my_email, get_upstream_commit, ROOT
+from kudu_util import check_output, confirm_prompt, Colors, get_my_email, get_upstream_commit, \
+ init_logging, ROOT
def check_repo_not_dirty():
@@ -190,5 +191,5 @@ def main():
if __name__ == "__main__":
- logging.basicConfig(level=logging.INFO)
+ init_logging()
main()
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/check_compatibility.py
----------------------------------------------------------------------
diff --git a/build-support/check_compatibility.py b/build-support/check_compatibility.py
index a8b8e3e..61d5661 100755
--- a/build-support/check_compatibility.py
+++ b/build-support/check_compatibility.py
@@ -32,7 +32,7 @@ import subprocess
import sys
import tempfile
-from kudu_util import check_output
+from kudu_util import check_output, init_logging
JAVA_ACC_GIT_URL = "https://github.com/lvc/japi-compliance-checker.git"
@@ -180,7 +180,6 @@ def run_java_acc(src_name, src, dst_name, dst):
def main(argv):
- logging.basicConfig(level=logging.INFO)
parser = optparse.OptionParser(
usage="usage: %prog SRC..[DST]")
parser.add_option("-f", "--force-download", dest="force_download_deps",
@@ -223,4 +222,5 @@ def main(argv):
if __name__ == "__main__":
+ init_logging()
main(sys.argv)
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/clang_tidy_gerrit.py
----------------------------------------------------------------------
diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py
index 3e77abe..d29a792 100755
--- a/build-support/clang_tidy_gerrit.py
+++ b/build-support/clang_tidy_gerrit.py
@@ -31,6 +31,8 @@ import sys
import unittest
import tempfile
+from kudu_util import init_logging
+
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
CLANG_TIDY_DIFF = os.path.join(
@@ -176,7 +178,7 @@ class TestClangTidyGerrit(unittest.TestCase):
if __name__ == "__main__":
# Basic setup and argument parsing.
- logging.basicConfig(level=logging.INFO)
+ init_logging()
parser = argparse.ArgumentParser(
description="Run clang-tidy on a patch, optionally posting warnings as comments to gerrit")
parser.add_argument("-n", "--no-gerrit", action="store_true",
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/dist_test.py
----------------------------------------------------------------------
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index 132b465..87ac820 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -39,6 +39,8 @@ import shutil
import subprocess
import time
+from kudu_util import init_logging
+
TEST_TIMEOUT_SECS = int(os.environ.get('TEST_TIMEOUT_SECS', '900'))
ARTIFACT_ARCHIVE_GLOBS = ["build/*/test-logs/**/*"]
ISOLATE_SERVER = os.environ.get('ISOLATE_SERVER',
@@ -165,7 +167,7 @@ def get_test_executions(options):
cwd=rel_to_abs("build/latest"))
out, err = p.communicate()
if p.returncode != 0:
- print >>sys.stderr, "Unable to list tests with ctest"
+ logging.error("Unable to list tests with ctest")
sys.exit(1)
lines = deque(out.splitlines())
execs = []
@@ -259,7 +261,7 @@ def ldd_deps(exe):
LDD_CACHE[exe] = (out, err, p.returncode)
out, err, rc = LDD_CACHE[exe]
if rc != 0:
- print >>sys.stderr, "failed to run ldd on ", exe
+ logging.warning("failed to run ldd on %s", exe)
return []
ret = []
for l in out.splitlines():
@@ -289,7 +291,7 @@ def create_archive_input(staging, execution,
"""
argv = execution.argv
if not argv[0].endswith('run-test.sh') or len(argv) < 2:
- print >>sys.stderr, "Unable to handle test: ", argv
+ logging.warning("Unable to handle test: %s", argv)
return
abs_test_exe = os.path.realpath(argv[1])
rel_test_exe = abs_to_rel(abs_test_exe, staging)
@@ -382,8 +384,8 @@ def create_task_json(staging,
}] * replicate_tasks
if len(tasks) > MAX_TASKS_PER_JOB:
- print >>sys.stderr, "Job contains %d tasks which is more than the maximum %d" % (
- len(tasks), MAX_TASKS_PER_JOB)
+ logging.error("Job contains %d tasks which is more than the maximum %d",
+ len(tasks), MAX_TASKS_PER_JOB)
sys.exit(1)
outmap = {"tasks": tasks}
@@ -406,7 +408,7 @@ def run_isolate(staging):
'-dump-json=' + staging.archive_dump_path(),
'--'] + staging.gen_json_paths())
except:
- print >>sys.stderr, "Failed to run", isolate_path
+ logging.error("Failed to run %s", isolate_path)
raise
def submit_tasks(staging, options):
@@ -418,9 +420,9 @@ def submit_tasks(staging, options):
by 'create_task_json()'.
"""
if not os.path.exists(DIST_TEST_HOME):
- print >>sys.stderr, "Cannot find dist_test tools at path %s " \
- "Set the DIST_TEST_HOME environment variable to the path to the dist_test directory. " \
- % DIST_TEST_HOME,
+ logging.error("Cannot find dist_test tools at path %s " \
+ "Set the DIST_TEST_HOME environment variable to the path to the dist_test directory. ",
+ DIST_TEST_HOME)
raise OSError("Cannot find path to dist_test tools")
client_py_path = os.path.join(DIST_TEST_HOME, "bin", "client")
try:
@@ -430,7 +432,7 @@ def submit_tasks(staging, options):
cmd.append(staging.tasks_json_path())
subprocess.check_call(cmd)
except:
- print >>sys.stderr, "Failed to run", client_py_path
+ logging.error("Failed to run %s", client_py_path)
raise
def get_flakies():
@@ -514,7 +516,6 @@ def add_loop_test_subparser(subparsers):
def main(argv):
- logging.basicConfig(level=logging.INFO)
p = argparse.ArgumentParser()
p.add_argument("--collect-tmpdir", dest="collect_tmpdir", action="store_true",
help="Collect the test tmpdir of failed tasks as test artifacts", default=False)
@@ -528,4 +529,5 @@ def main(argv):
if __name__ == "__main__":
+ init_logging()
main(sys.argv[1:])
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/gen_version_info.py
----------------------------------------------------------------------
diff --git a/build-support/gen_version_info.py b/build-support/gen_version_info.py
index d8aa88a..fbb0985 100755
--- a/build-support/gen_version_info.py
+++ b/build-support/gen_version_info.py
@@ -30,7 +30,7 @@ import sys
import time
from time import strftime, localtime
-from kudu_util import check_output
+from kudu_util import check_output, init_logging
def output_up_to_date(path, id_hash):
"""
@@ -46,7 +46,6 @@ def output_up_to_date(path, id_hash):
return m.group(1) == id_hash
def main():
- logging.basicConfig(level=logging.INFO)
parser = optparse.OptionParser(
usage="usage: %prog --version=<version> <output path>")
parser.add_option("-v", "--version", help="Set version number", type="string",
@@ -127,5 +126,7 @@ def main():
""" % locals())
return 0
+
if __name__ == "__main__":
+ init_logging()
sys.exit(main())
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/iwyu.py
----------------------------------------------------------------------
diff --git a/build-support/iwyu.py b/build-support/iwyu.py
index 4fdcbdd..2638cdd 100755
--- a/build-support/iwyu.py
+++ b/build-support/iwyu.py
@@ -28,7 +28,7 @@ import re
import subprocess
import sys
-from kudu_util import get_upstream_commit, check_output, ROOT, Colors
+from kudu_util import get_upstream_commit, check_output, ROOT, Colors, init_logging
import iwyu.fix_includes
from iwyu.fix_includes import ParseAndMergeIWYUOutput
@@ -268,7 +268,6 @@ def main(argv):
else:
return _do_iwyu(flags, paths)
-
if __name__ == "__main__":
- logging.basicConfig(level=logging.INFO)
+ init_logging()
sys.exit(main(sys.argv))
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/kudu_util.py
----------------------------------------------------------------------
diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
index eac5115..15d1059 100644
--- a/build-support/kudu_util.py
+++ b/build-support/kudu_util.py
@@ -22,6 +22,7 @@
from __future__ import print_function
+import logging
import os
import subprocess
import sys
@@ -42,6 +43,19 @@ def get_upstream_commit():
""" Return the last commit hash that appears to have been committed by gerrit. """
return check_output(_GET_UPSTREAM_COMMIT_SCRIPT).strip().decode('utf-8')
+def init_logging():
+ logging.basicConfig(level=logging.INFO,
+ format='%(asctime)s %(levelname)s: %(message)s')
+ logging.getLogger().addFilter(ColorFilter())
+
+class ColorFilter(logging.Filter):
+ """ logging.Filter implementation which colorizes the output to console. """
+ def filter(self, record):
+ if record.levelno >= logging.ERROR:
+ record.msg = Colors.RED + record.msg + Colors.RESET
+ elif record.levelno >= logging.WARNING:
+ record.msg = Colors.YELLOW + record.msg + Colors.RESET
+ return True
class Colors(object):
""" ANSI color codes. """
http://git-wip-us.apache.org/repos/asf/kudu/blob/953a7070/build-support/push_to_asf.py
----------------------------------------------------------------------
diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py
index 5b4d15f..1752dbe 100755
--- a/build-support/push_to_asf.py
+++ b/build-support/push_to_asf.py
@@ -40,7 +40,7 @@ import re
import subprocess
import sys
-from kudu_util import check_output, confirm_prompt, Colors, get_my_email
+from kudu_util import check_output, confirm_prompt, Colors, get_my_email, init_logging
APACHE_REPO = "https://git-wip-us.apache.org/repos/asf/kudu.git"
GERRIT_URL = "ssh://<username>@gerrit.cloudera.org:29418/kudu"
@@ -235,5 +235,5 @@ def main():
if __name__ == "__main__":
- logging.basicConfig(level=logging.INFO)
+ init_logging()
main()