impala git commit: test-with-docker: add --env option to pass through env variables

[ta...@apache.org]

Repository: impala
Updated Branches:
  refs/heads/master 8cbec20ea -> 1dbb3c1be


test-with-docker: add --env option to pass through env variables

Adds simple mechanism to pass environment variables into
test-with-docker, which is occasionally useful, especially for
development and tinkering with tests. It's typically the right thing to
codify the environment variables into tests, but a pass through can be
handy.

The implementation is simple enough, passing the
variables into the docker containers.

Change-Id: I03c2feda8edc2983e423f862ed210fabb845714f
Reviewed-on: http://gerrit.cloudera.org:8080/11730
Reviewed-by: Laszlo Gaal <la...@cloudera.com>
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/1dbb3c1b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/1dbb3c1b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/1dbb3c1b

Branch: refs/heads/master
Commit: 1dbb3c1be634c2a3e356c3b2ee5deac2436c9815
Parents: 8cbec20
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Wed May 16 12:23:47 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Oct 25 23:52:58 2018 +0000

----------------------------------------------------------------------
 docker/test-with-docker.py | 44 ++++++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/1dbb3c1b/docker/test-with-docker.py
----------------------------------------------------------------------
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index 4efd712..88f4563 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -18,6 +18,23 @@
 # under the License.
 #
 
+# We do not use Impala's python environment here, nor do we depend on
+# non-standard python libraries to avoid needing extra build steps before
+# triggering this.
+import argparse
+import datetime
+import itertools
+import logging
+import multiprocessing
+import multiprocessing.pool
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
 CLI_HELP = """\
 Runs tests inside of docker containers, parallelizing different types of
 tests. This script first creates a docker container, checks out this repo
@@ -100,22 +117,6 @@ as part of the build, in logs/docker/*/timeline.html.
 #     or move to different suite.
 #   - Run BE tests earlier (during data load)
 
-# We do not use Impala's python environment here, nor do we depend on
-# non-standard python libraries to avoid needing extra build steps before
-# triggering this.
-import argparse
-import datetime
-import logging
-import multiprocessing
-import multiprocessing.pool
-import os
-import re
-import subprocess
-import sys
-import tempfile
-import threading
-import time
-
 if __name__ == '__main__' and __package__ is None:
   sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
   import monitor
@@ -181,6 +182,10 @@ def main():
                       default=os.path.expanduser("~/.ccache"))
   parser.add_argument('--tail', action="store_true",
       help="Run tail on all container log files.")
+  parser.add_argument('--env', metavar='K=V', default=[], action='append',
+      help="""Passes given environment variables (expressed as KEY=VALUE)
+           through containers.
+           """)
   parser.add_argument('--test', action="store_true")
   args = parser.parse_args()
 
@@ -197,7 +202,8 @@ def main():
       parallel_test_concurrency=args.parallel_test_concurrency,
       suite_concurrency=args.suite_concurrency,
       impalad_mem_limit_bytes=args.impalad_mem_limit_bytes,
-      tail=args.tail)
+      tail=args.tail,
+      env=args.env)
 
   fh = logging.FileHandler(os.path.join(_make_dir_if_not_exist(t.log_dir), "log.txt"))
   fh.setFormatter(logging.Formatter(LOG_FORMAT))
@@ -430,7 +436,7 @@ class TestWithDocker(object):
   def __init__(self, build_image, suite_names, name, cleanup_containers,
                cleanup_image, ccache_dir, test_mode,
                suite_concurrency, parallel_test_concurrency,
-               impalad_mem_limit_bytes, tail):
+               impalad_mem_limit_bytes, tail, env):
     self.build_image = build_image
     self.name = name
     self.containers = []
@@ -466,6 +472,7 @@ class TestWithDocker(object):
     self.parallel_test_concurrency = parallel_test_concurrency
     self.impalad_mem_limit_bytes = impalad_mem_limit_bytes
     self.tail = tail
+    self.env = env
 
     # Map suites back into objects; we ignore case for this mapping.
     suites = []
@@ -545,6 +552,7 @@ class TestWithDocker(object):
           "-v", _make_dir_if_not_exist(self.log_dir,
                                        logdir) + ":/logs",
           "-v", base + ":/mnt/base:ro"]
+          + list(itertools.chain(*[["-e", env] for env in self.env]))
           + extras
           + [image]
           + entrypoint).strip()