[3/3] impala git commit: test-with-docker: allow built images to be used with "docker run" easily.

[mi...@apache.org]

test-with-docker: allow built images to be used with "docker run" easily.

Configures the built container to enter into a script that
starts the minicluster. As a result, "docker run -ti <container>" will
launch the user into a shell with the Impala minicluster and the
impala development cluster running.

To handle cases where users don't specify --privileged, we skip
Kudu if it NTP seems unavailable.

Change-Id: Ib8d6a28d4cb4ab019cd72415024b23374a6d9e2f
Reviewed-on: http://gerrit.cloudera.org:8080/11781
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/de0c6bd6
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/de0c6bd6
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/de0c6bd6

Branch: refs/heads/master
Commit: de0c6bd6bd0db163d2820ae238bee5887f410f52
Parents: c170107
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Mon Oct 22 21:38:14 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Oct 26 18:44:58 2018 +0000

----------------------------------------------------------------------
 docker/entrypoint.sh       | 65 ++++++++++++++++++++++++++++++++++-------
 docker/test-with-docker.py |  3 ++
 2 files changed, 58 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/de0c6bd6/docker/entrypoint.sh
----------------------------------------------------------------------
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 1dbc6c1..50a38bd 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -145,16 +145,18 @@ function start_minicluster {
   # presumably because there's only one layer involved. See
   # https://issues.apache.org/jira/browse/KUDU-1419.
   set -x
-  pushd /home/impdev/Impala/testdata
-  for x in cluster/cdh*/node-*/var/lib/kudu/*/wal; do
-    echo $x
-    # This mv takes time, as it's actually copying into the latest layer.
-    mv $x $x-orig
-    mkdir $x
-    mv $x-orig/* $x
-    rmdir $x-orig
-  done
-  popd
+  if [ "true" = $KUDU_IS_SUPPORTED ]; then
+    pushd /home/impdev/Impala/testdata
+    for x in cluster/cdh*/node-*/var/lib/kudu/*/wal; do
+      echo $x
+      # This mv takes time, as it's actually copying into the latest layer.
+      mv $x $x-orig
+      mkdir $x
+      mv $x-orig/* $x
+      rmdir $x-orig
+    done
+    popd
+  fi
 
   # Wait for postgresql to really start; if it doesn't, Hive Metastore will fail to start.
   for i in {1..120}; do
@@ -387,6 +389,42 @@ function configure_timezone() {
   fi
 }
 
+# Exposes a shell, with the container booted with
+# a minicluster.
+function shell() {
+  echo "Starting minicluster and Impala."
+  # Logs is typically a symlink; remove it if so.
+  rm logs || true
+  mkdir -p logs
+  boot_container
+  impala_environment
+  # Kudu requires --privileged for the Docker container; see
+  # https://issues.apache.org/jira/browse/KUDU-2000. Because
+  # our goal here is convenience for new developers, we
+  # skip kudu if "ntptime" doesn't work, which is a good
+  # proxy for Kudu won't start.
+  if ! ntptime > /dev/null; then
+    export KUDU_IS_SUPPORTED=false
+    KUDU_MSG="Kudu is not started."
+  fi
+  start_minicluster
+  bin/start-impala-cluster.py
+  cat <<"EOF"
+
+==========================================================
+Welcome to the Impala development environment.
+
+The "minicluster" is running; i.e., HDFS, HBase, Hive,
+etc. are running. $KUDU_MSG
+
+To get started, perhaps run:
+  impala-shell.sh -q 'select count(*) from tpcds.web_page'
+==========================================================
+
+EOF
+  exec bash
+}
+
 function main() {
   set -e
 
@@ -394,6 +432,13 @@ function main() {
   CMD="$1"
   shift
 
+  # Treat shell specialy to avoid the extra logging and | cat below.
+  if [[ $CMD = "shell" ]]; then
+    shell
+    # shell shoud have exec'd, so if we get here, it's a failure.
+    exit 1
+  fi
+
   echo ">>> ${CMD} $@ (begin)"
   # Dump environment, for debugging
   env | grep -vE "AWS_(SECRET_)?ACCESS_KEY"

http://git-wip-us.apache.org/repos/asf/impala/blob/de0c6bd6/docker/test-with-docker.py
----------------------------------------------------------------------
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index b350e4c..42808cf 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -653,6 +653,9 @@ class TestWithDocker(object):
       self.image = _check_output(
           ["docker", "commit",
            "-c", "LABEL pwd=" + self.git_root,
+           "-c", "USER impdev",
+           "-c", "WORKDIR /home/impdev/Impala",
+           "-c", 'CMD ["/home/impdev/Impala/docker/entrypoint.sh", "shell"]',
            container.id, "impala:built-" + self.name]).strip()
       logging.info("Committed docker image: %s", self.image)
     finally: