[impala] 01/02: fe: set classpath using maven dependency resolution

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 559f19a5be95d686a71e616517ad4ae916c85824
Author: Todd Lipcon <to...@apache.org>
AuthorDate: Mon Apr 29 20:34:38 2019 -0700

    fe: set classpath using maven dependency resolution
    
    This changes the FE pom to generate a build classpath file in the
    target/ directory. Then, bin/set-classpath.sh uses this file to generate
    the classpath to start the cluster. This replaces the former approach of
    including all of the jars found in target/dependency/
    
    The advantage of this is that a clean build is no longer required when
    switching artifact versions. Prior to this patch, if you changed an
    artifact version and rebuilt, both the old and new artifact would be
    left in the target/dependency/ directory and pollute the classpath.
    
    This doesn't fully remove the target/dependency/ directory, because its
    existence is likely important for downstream packaging of Impala. We can
    likely assume that such packaging always does a clean build.
    
    This also changes the set-classpath script to no longer load jars from
    testdata/target/dependency/ since it appears that directory doesn't
    actually get created during the build.
    
    Change-Id: I103a1da10a54c7525ba7fb584d942ba1cb9fcb94
    Reviewed-on: http://gerrit.cloudera.org:8080/13185
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
    Reviewed-by: Todd Lipcon <to...@apache.org>
---
 bin/set-classpath.sh          | 18 ++++++++----------
 docker/setup_build_context.py | 14 +++++++++-----
 fe/pom.xml                    | 20 +++++++++++++++++++-
 3 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh
index 68aec2b..0f98533 100755
--- a/bin/set-classpath.sh
+++ b/bin/set-classpath.sh
@@ -31,16 +31,14 @@ CLASSPATH=\
 "${HIVE_HOME}"/lib/datanucleus-core-3.2.2.jar:\
 "${HIVE_HOME}"/lib/datanucleus-rdbms-3.2.1.jar:
 
-for jar in "${IMPALA_HOME}"/fe/target/dependency/*.jar; do
-  if [ -e "$jar" ] ; then
-    CLASSPATH="${CLASSPATH}:$jar"
-  fi
-done
+FE_CP_FILE="$IMPALA_HOME/fe/target/build-classpath.txt"
 
-for jar in "${IMPALA_HOME}"/testdata/target/dependency/*.jar; do
-  if [ -e "$jar" ] ; then
-    CLASSPATH="${CLASSPATH}:$jar"
-  fi
-done
+if [ ! -s "$FE_CP_FILE" ]; then
+  >&2 echo FE classpath file $FE_CP_FILE missing.
+  >&2 echo Build the front-end first.
+  exit 1
+fi
+
+CLASSPATH=$(cat "$IMPALA_HOME"/fe/target/build-classpath.txt):"$CLASSPATH"
 
 export CLASSPATH
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index 0b50fb6..817263e 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -60,11 +60,15 @@ for lib in ["libstdc++", "libgcc"]:
         symlink_file_into_dir(so, LIB_DIR)
 os.symlink(os.environ["IMPALA_KUDU_HOME"], os.path.join(OUTPUT_DIR, "kudu"))
 
-# Impala jars and dependencies.
-for glob_pattern in [os.path.join(IMPALA_HOME, "fe/target/dependency/*.jar"),
-    os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")]:
-  for jar in glob.glob(glob_pattern):
-      symlink_file_into_dir(jar, LIB_DIR)
+# Impala dependencies.
+dep_classpath = file(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
+for jar in dep_classpath.split(":"):
+  assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
+  symlink_file_into_dir(jar, LIB_DIR)
+
+# Impala jars.
+for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")):
+  symlink_file_into_dir(jar, LIB_DIR)
 
 # Templates for debug web pages.
 os.symlink(os.path.join(IMPALA_HOME, "www"), os.path.join(OUTPUT_DIR, "www"))
diff --git a/fe/pom.xml b/fe/pom.xml
index bdab1b7..62fc500 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -492,8 +492,11 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>2.10</version>
+        <version>3.1.1</version>
         <executions>
+          <!-- TODO(todd): consider removing this execution or moving it to
+               some kind of 'dist' profile. No need to copy all of these jars
+               on every build of the FE! -->
           <execution>
             <id>copy-dependencies</id>
             <phase>package</phase>
@@ -506,6 +509,21 @@ under the License.
               <silent>true</silent>
             </configuration>
           </execution>
+          <!--
+            Write the runtime classpath to a file in the target directory
+            so it can be picked up by bin/set-classpath.sh
+          -->
+          <execution>
+            <id>write-classpath</id>
+            <goals>
+              <goal>build-classpath</goal>
+            </goals>
+            <configuration>
+              <outputFile>${project.build.directory}/build-classpath.txt</outputFile>
+              <includeScope>runtime</includeScope>
+              <excludeTypes>pom</excludeTypes>
+            </configuration>
+          </execution>
         </executions>
       </plugin>