You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/10/15 19:30:41 UTC
[impala] branch master updated (398f17f -> 97792c4)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 398f17f IMPALA-9440 Typo in rpcz.tmpl for inbound connection metrics
new 9785647 IMPALA-10198 (part 1): Unify Java in a single java/ directory
new 97792c4 IMPALA-10198 (part 2): Add support for mvn versions:set
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 9 +----
bin/clean-cmake.sh | 4 +--
bin/clean.sh | 2 +-
bin/create_testdata.sh | 6 ++--
bin/jenkins/build-all-flag-combinations.sh | 42 +++++++++++++++++++++-
buildall.sh | 4 +--
common/thrift/CMakeLists.txt | 2 +-
common/yarn-extras/CMakeLists.txt | 23 ------------
docker/CMakeLists.txt | 2 +-
docker/setup_build_context.py | 12 +++++--
ext-data-source/CMakeLists.txt | 23 ------------
fe/CMakeLists.txt | 22 ------------
fe/pom.xml | 6 ++--
{impala-parent => java}/CMakeLists.txt | 2 +-
{testdata => java}/TableFlattener/.gitignore | 0
{testdata => java}/TableFlattener/README | 0
{testdata => java}/TableFlattener/pom.xml | 5 +--
.../impala/infra/tableflattener/FileMigrator.java | 0
.../infra/tableflattener/FlattenedSchema.java | 0
.../apache/impala/infra/tableflattener/Main.java | 0
.../infra/tableflattener/SchemaFlattener.java | 0
.../impala/infra/tableflattener/SchemaUtil.java | 0
{testdata => java/datagenerator}/pom.xml | 9 ++---
.../impala/datagenerator/CsvToHBaseConverter.java | 0
.../datagenerator/JsonToParquetConverter.java | 0
.../datagenerator/RandomNestedDataGenerator.java | 0
.../impala/datagenerator/TestDataGenerator.java | 0
{mvn-deps => java}/executor-deps/pom.xml | 15 ++++++--
.../ext-data-source}/.gitignore | 0
.../ext-data-source}/api/pom.xml | 4 +--
.../extdatasource/util/SerializationUtils.java | 0
.../extdatasource/v1/ExternalDataSource.java | 0
{ext-data-source => java/ext-data-source}/pom.xml | 5 +--
.../ext-data-source}/sample/pom.xml | 4 +--
.../extdatasource/sample/EchoDataSource.java | 0
.../ext-data-source}/test/pom.xml | 8 +----
.../impala/extdatasource/AllTypesDataSource.java | 0
{impala-parent => java}/pom.xml | 19 ++++++++--
.../query-event-hook-api}/pom.xml | 5 +--
.../apache/impala/hooks/QueryCompleteContext.java | 0
.../org/apache/impala/hooks/QueryEventHook.java | 0
{mvn-deps => java}/shaded-deps/hive-exec/pom.xml | 5 ++-
{mvn-deps => java}/shaded-deps/s3a-aws-sdk/pom.xml | 5 ++-
{tests => java}/test-hive-udfs/pom.xml | 4 +--
.../org/apache/impala/ImportsNearbyClassesUdf.java | 0
.../org/apache/impala/IncompatibleUdfTest.java | 0
.../main/java/org/apache/impala/JavaGcUdfTest.java | 0
.../main/java/org/apache/impala/JavaUdfTest.java | 0
.../java/org/apache/impala/ReplaceStringUdf.java | 0
.../src/main/java/org/apache/impala/TestUdf.java | 0
.../java/org/apache/impala/TestUdfException.java | 0
.../main/java/org/apache/impala/TestUpdateUdf.java | 0
.../main/java/org/apache/impala/UnresolvedUdf.java | 0
.../main/java/org/apache/impala/UtilForUdf.java | 0
{common => java}/yarn-extras/.gitignore | 0
{common => java}/yarn-extras/README.txt | 0
{common => java}/yarn-extras/pom.xml | 32 ++++++++++++++---
.../resourcemanager/resource/ResourceWeights.java | 0
.../scheduler/fair/AllocationConfiguration.java | 0
.../fair/AllocationConfigurationException.java | 0
.../fair/AllocationFileLoaderService.java | 0
.../scheduler/fair/FSQueueType.java | 0
.../scheduler/fair/FairSchedulerConfiguration.java | 0
.../scheduler/fair/QueuePlacementPolicy.java | 0
.../scheduler/fair/QueuePlacementRule.java | 0
.../scheduler/fair/SchedulingPolicy.java | 0
.../impala/yarn/server/utils/BuilderUtils.java | 0
mvn-deps/executor-deps/CMakeLists.txt | 20 -----------
mvn-deps/shaded-deps/hive-exec/CMakeLists.txt | 20 -----------
mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt | 20 -----------
query-event-hook-api/CMakeLists.txt | 22 ------------
testdata/bin/copy-data-sources.sh | 2 +-
testdata/bin/copy-udfs-udas.sh | 8 +++--
testdata/bin/create-load-data.sh | 2 +-
testdata/bin/generate-load-nested.sh | 6 ++--
tests/custom_cluster/test_coordinators.py | 2 +-
tests/query_test/test_udfs.py | 4 +--
tests/run-tests.py | 2 +-
78 files changed, 150 insertions(+), 237 deletions(-)
delete mode 100644 common/yarn-extras/CMakeLists.txt
delete mode 100644 ext-data-source/CMakeLists.txt
delete mode 100644 fe/CMakeLists.txt
rename {impala-parent => java}/CMakeLists.txt (93%)
rename {testdata => java}/TableFlattener/.gitignore (100%)
rename {testdata => java}/TableFlattener/README (100%)
rename {testdata => java}/TableFlattener/pom.xml (93%)
rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java (100%)
rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java (100%)
rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java (100%)
rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java (100%)
rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java (100%)
rename {testdata => java/datagenerator}/pom.xml (94%)
rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java (100%)
rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java (100%)
rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java (100%)
rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java (100%)
rename {mvn-deps => java}/executor-deps/pom.xml (95%)
rename {ext-data-source => java/ext-data-source}/.gitignore (100%)
rename {ext-data-source => java/ext-data-source}/api/pom.xml (96%)
rename {ext-data-source => java/ext-data-source}/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java (100%)
rename {ext-data-source => java/ext-data-source}/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java (100%)
rename {ext-data-source => java/ext-data-source}/pom.xml (89%)
rename {ext-data-source => java/ext-data-source}/sample/pom.xml (95%)
rename {ext-data-source => java/ext-data-source}/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java (100%)
rename {ext-data-source => java/ext-data-source}/test/pom.xml (90%)
rename {ext-data-source => java/ext-data-source}/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java (100%)
rename {impala-parent => java}/pom.xml (93%)
rename {query-event-hook-api => java/query-event-hook-api}/pom.xml (92%)
rename {query-event-hook-api => java/query-event-hook-api}/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java (100%)
rename {query-event-hook-api => java/query-event-hook-api}/src/main/java/org/apache/impala/hooks/QueryEventHook.java (100%)
rename {mvn-deps => java}/shaded-deps/hive-exec/pom.xml (97%)
rename {mvn-deps => java}/shaded-deps/s3a-aws-sdk/pom.xml (98%)
rename {tests => java}/test-hive-udfs/pom.xml (95%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java (100%)
rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java (100%)
rename {common => java}/yarn-extras/.gitignore (100%)
rename {common => java}/yarn-extras/README.txt (100%)
rename {common => java}/yarn-extras/pom.xml (76%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java (100%)
rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java (100%)
delete mode 100644 mvn-deps/executor-deps/CMakeLists.txt
delete mode 100644 mvn-deps/shaded-deps/hive-exec/CMakeLists.txt
delete mode 100644 mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt
delete mode 100644 query-event-hook-api/CMakeLists.txt
[impala] 01/02: IMPALA-10198 (part 1): Unify Java in a single java/
directory
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 97856478ec4f952486786633d10e02d0682a0f94
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Oct 6 11:03:33 2020 -0700
IMPALA-10198 (part 1): Unify Java in a single java/ directory
This changes all existing Java code to be submodules under
a single root pom. The root pom is impala-parent/pom.xml
with minor changes to add submodules.
This avoids most of the weird CMake/maven interactions,
because there is now a single maven invocation for all
the Java code.
This moves all the Java projects other than fe into
a top level java directory. fe is left where it is
to avoid disruption (but still is compiled via the
java directory's root pom). Various pieces of code
that reference the old locations are updated.
Based on research, there are two options for dealing
with the shaded dependencies. The first is to have an
entirely separate Maven project with a separate Maven
invocation. In this case, the consumers of the shaded
jars will see the reduced set of transitive dependencies.
The second is to have the shaded dependencies as modules
with a single Maven invocation. The consumer would see
all of the original transitive dependencies and need to
exclude them all. See MSHADE-206/MNG-5899. This chooses
the second.
This only moves code around and does not focus on version
numbers or making "mvn versions:set" work.
Testing:
- Ran a core job
- Verified existing maven commands from fe/ directory still work
- Compared the *-classpath.txt files from fe and executor-deps
and verified they are the same except for paths
Change-Id: I08773f4f9d7cb269b0491080078d6e6f490d8d7a
Reviewed-on: http://gerrit.cloudera.org:8080/16500
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
CMakeLists.txt | 9 +--------
bin/clean-cmake.sh | 4 ++--
bin/clean.sh | 2 +-
bin/create_testdata.sh | 6 ++++--
buildall.sh | 4 ++--
common/thrift/CMakeLists.txt | 2 +-
common/yarn-extras/CMakeLists.txt | 23 ----------------------
docker/CMakeLists.txt | 2 +-
docker/setup_build_context.py | 2 +-
ext-data-source/CMakeLists.txt | 23 ----------------------
fe/CMakeLists.txt | 22 ---------------------
fe/pom.xml | 2 +-
{impala-parent => java}/CMakeLists.txt | 2 +-
{testdata => java}/TableFlattener/.gitignore | 0
{testdata => java}/TableFlattener/README | 0
{testdata => java}/TableFlattener/pom.xml | 1 -
.../impala/infra/tableflattener/FileMigrator.java | 0
.../infra/tableflattener/FlattenedSchema.java | 0
.../apache/impala/infra/tableflattener/Main.java | 0
.../infra/tableflattener/SchemaFlattener.java | 0
.../impala/infra/tableflattener/SchemaUtil.java | 0
{testdata => java/datagenerator}/pom.xml | 1 -
.../impala/datagenerator/CsvToHBaseConverter.java | 0
.../datagenerator/JsonToParquetConverter.java | 0
.../datagenerator/RandomNestedDataGenerator.java | 0
.../impala/datagenerator/TestDataGenerator.java | 0
{mvn-deps => java}/executor-deps/pom.xml | 13 +++++++++++-
.../ext-data-source}/.gitignore | 0
.../ext-data-source}/api/pom.xml | 0
.../extdatasource/util/SerializationUtils.java | 0
.../extdatasource/v1/ExternalDataSource.java | 0
{ext-data-source => java/ext-data-source}/pom.xml | 1 -
.../ext-data-source}/sample/pom.xml | 0
.../extdatasource/sample/EchoDataSource.java | 0
.../ext-data-source}/test/pom.xml | 0
.../impala/extdatasource/AllTypesDataSource.java | 0
{impala-parent => java}/pom.xml | 13 ++++++++++++
.../query-event-hook-api}/pom.xml | 1 -
.../apache/impala/hooks/QueryCompleteContext.java | 0
.../org/apache/impala/hooks/QueryEventHook.java | 0
{mvn-deps => java}/shaded-deps/hive-exec/pom.xml | 2 +-
{mvn-deps => java}/shaded-deps/s3a-aws-sdk/pom.xml | 2 +-
{tests => java}/test-hive-udfs/pom.xml | 1 -
.../org/apache/impala/ImportsNearbyClassesUdf.java | 0
.../org/apache/impala/IncompatibleUdfTest.java | 0
.../main/java/org/apache/impala/JavaGcUdfTest.java | 0
.../main/java/org/apache/impala/JavaUdfTest.java | 0
.../java/org/apache/impala/ReplaceStringUdf.java | 0
.../src/main/java/org/apache/impala/TestUdf.java | 0
.../java/org/apache/impala/TestUdfException.java | 0
.../main/java/org/apache/impala/TestUpdateUdf.java | 0
.../main/java/org/apache/impala/UnresolvedUdf.java | 0
.../main/java/org/apache/impala/UtilForUdf.java | 0
{common => java}/yarn-extras/.gitignore | 0
{common => java}/yarn-extras/README.txt | 0
{common => java}/yarn-extras/pom.xml | 1 -
.../resourcemanager/resource/ResourceWeights.java | 0
.../scheduler/fair/AllocationConfiguration.java | 0
.../fair/AllocationConfigurationException.java | 0
.../fair/AllocationFileLoaderService.java | 0
.../scheduler/fair/FSQueueType.java | 0
.../scheduler/fair/FairSchedulerConfiguration.java | 0
.../scheduler/fair/QueuePlacementPolicy.java | 0
.../scheduler/fair/QueuePlacementRule.java | 0
.../scheduler/fair/SchedulingPolicy.java | 0
.../impala/yarn/server/utils/BuilderUtils.java | 0
mvn-deps/executor-deps/CMakeLists.txt | 20 -------------------
mvn-deps/shaded-deps/hive-exec/CMakeLists.txt | 20 -------------------
mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt | 20 -------------------
query-event-hook-api/CMakeLists.txt | 22 ---------------------
testdata/bin/copy-data-sources.sh | 2 +-
testdata/bin/copy-udfs-udas.sh | 8 ++++++--
testdata/bin/create-load-data.sh | 2 +-
testdata/bin/generate-load-nested.sh | 6 +++---
tests/custom_cluster/test_coordinators.py | 2 +-
tests/query_test/test_udfs.py | 4 ++--
tests/run-tests.py | 2 +-
77 files changed, 57 insertions(+), 190 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f88a68d..ddfc26b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -425,17 +425,10 @@ endif()
add_subdirectory(common/function-registry)
add_subdirectory(common/thrift)
add_subdirectory(common/fbs)
-add_subdirectory(common/yarn-extras)
add_subdirectory(common/protobuf)
add_subdirectory(be)
add_subdirectory(docker)
-add_subdirectory(mvn-deps/shaded-deps/hive-exec)
-add_subdirectory(mvn-deps/shaded-deps/s3a-aws-sdk)
-add_subdirectory(mvn-deps/executor-deps)
-add_subdirectory(fe)
-add_subdirectory(impala-parent)
-add_subdirectory(ext-data-source)
-add_subdirectory(query-event-hook-api)
+add_subdirectory(java)
# Build target for all generated files which most backend code depends on
add_custom_target(gen-deps ALL DEPENDS thrift-deps proto-deps fb-deps
diff --git a/bin/clean-cmake.sh b/bin/clean-cmake.sh
index aa1ea96..bd9cc77 100755
--- a/bin/clean-cmake.sh
+++ b/bin/clean-cmake.sh
@@ -29,8 +29,8 @@ if [[ -z "${IMPALA_HOME}" || ! -d "${IMPALA_HOME}" ]]; then
fi
# Remove trailing /
ROOT_DIR=${IMPALA_HOME%%/}
-for loc in "${ROOT_DIR}/ -maxdepth 1" "${ROOT_DIR}/be/" "${ROOT_DIR}/fe/" "${ROOT_DIR}/common/"\
- "${ROOT_DIR}/ext-data-source/"; do
+for loc in "${ROOT_DIR}/ -maxdepth 1" "${ROOT_DIR}/be/" "${ROOT_DIR}/java/" \
+ "${ROOT_DIR}/common/" ; do
find ${loc} \( -iname CMakeCache.txt -o -iname CMakeFiles \
-o -iname CTestTestfile.cmake -o -iname cmake_install.cmake \) -exec rm -Rf {} +
done
diff --git a/bin/clean.sh b/bin/clean.sh
index 34781f9..259648a 100755
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -30,7 +30,7 @@ setup_report_build_error
"${MAKE_CMD:-make}" clean || :
# clean the external data source project
-pushd "${IMPALA_HOME}/ext-data-source"
+pushd "${IMPALA_HOME}/java/ext-data-source"
rm -rf api/generated-sources/*
${IMPALA_HOME}/bin/mvn-quiet.sh clean
popd
diff --git a/bin/create_testdata.sh b/bin/create_testdata.sh
index 5b81d9e..9c70362 100755
--- a/bin/create_testdata.sh
+++ b/bin/create_testdata.sh
@@ -29,13 +29,13 @@ bin=`cd "$bin"; pwd`
DATALOC=$IMPALA_HOME/testdata/target
# regenerate the test data generator
-cd $IMPALA_HOME/testdata
+cd $IMPALA_HOME/java/datagenerator
${IMPALA_HOME}/bin/mvn-quiet.sh clean
${IMPALA_HOME}/bin/mvn-quiet.sh package
# find jars
CP=""
-JARS=`find target/*.jar 2> /dev/null || true`
+JARS=`find ${IMPALA_HOME}/java/datagenerator/target/*.jar 2> /dev/null || true`
for i in $JARS; do
if [ -n "$CP" ]; then
CP=${CP}:${i}
@@ -47,6 +47,8 @@ done
# run test data generator
echo $DATALOC
mkdir -p $DATALOC
+pushd $IMPALA_HOME/testdata
"$JAVA" -cp $CP org.apache.impala.datagenerator.TestDataGenerator $DATALOC
"$JAVA" -cp $CP org.apache.impala.datagenerator.CsvToHBaseConverter
echo "SUCCESS, data generated into $DATALOC"
+popd
diff --git a/buildall.sh b/buildall.sh
index 1440e32..2a7eade 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -424,7 +424,7 @@ bootstrap_dependencies() {
# Build the Impala frontend and its dependencies.
build_fe() {
generate_cmake_files $CMAKE_BUILD_TYPE
- ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} fe
+ ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} java
}
# Build all components. The build type is specified as the first argument, and the
@@ -449,7 +449,7 @@ build_all_components() {
MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf \
udasample udfsample"
if (( build_independent_targets )); then
- MAKE_TARGETS+=" cscope fe tarballs"
+ MAKE_TARGETS+=" cscope java tarballs"
fi
fi
${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} ${IMPALA_MAKE_FLAGS} ${MAKE_TARGETS}
diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt
index 9e7be77..9b14ab5 100644
--- a/common/thrift/CMakeLists.txt
+++ b/common/thrift/CMakeLists.txt
@@ -177,7 +177,7 @@ set(THRIFT_INCLUDE_DIR_OPTION -I ${THRIFT_CONTRIB_DIR} -I $ENV{HIVE_METASTORE_TH
set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources)
set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources)
# TODO: avoid duplicating generated java classes
-set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/ext-data-source/api/generated-sources)
+set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/java/ext-data-source/api/generated-sources)
set(PYTHON_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/shell/)
set(THRIFT11_PYTHON_OUTPUT_DIR ${PYTHON_OUTPUT_DIR}/build/thrift-11-gen)
MESSAGE("Found output dir: " ${PYTHON_OUTPUT_DIR})
diff --git a/common/yarn-extras/CMakeLists.txt b/common/yarn-extras/CMakeLists.txt
deleted file mode 100644
index 4f46ba5..0000000
--- a/common/yarn-extras/CMakeLists.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The dependency on ext-data-source here is fictional, but Maven does not like
-# concurrent invocations. These lead to opaque, non-deterministic errors due to
-# races in how Maven handles its ~/.m2/repository doc.
-add_custom_target(yarn-extras ALL DEPENDS impala-parent ext-data-source
- COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index 44f1329..dfb6a33 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -48,7 +48,7 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# with symlinks
add_custom_target(impala_base_build_context_${build_type}
COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args}
- DEPENDS daemons fe ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
+ DEPENDS daemons java ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
COMMENT "Creating impala base build context build_type=${build_type}."
VERBATIM
)
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index f359328..f41a757 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -137,7 +137,7 @@ for jar in glob.glob(
# Impala Executor dependencies.
dep_classpath = file(os.path.join(IMPALA_HOME,
- "mvn-deps/executor-deps/target/build-executor-deps-classpath.txt")).read()
+ "java/executor-deps/target/build-executor-deps-classpath.txt")).read()
for jar in dep_classpath.split(":"):
assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
symlink_file_into_dir(jar, EXEC_LIB_DIR)
diff --git a/ext-data-source/CMakeLists.txt b/ext-data-source/CMakeLists.txt
deleted file mode 100644
index 0f88dd2..0000000
--- a/ext-data-source/CMakeLists.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The dependency on shaded-deps is only added to avoid parallel downloads
-# of dependencies. For more details see IMPALA-7051, which was a similar issue.
-add_custom_target(ext-data-source ALL DEPENDS gen-deps impala-parent
- shaded-deps-hive-exec shaded-deps-s3a-aws-sdk
- COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/fe/CMakeLists.txt b/fe/CMakeLists.txt
deleted file mode 100644
index 2d52ade..0000000
--- a/fe/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(fe ALL DEPENDS
- executor-deps shaded-deps-hive-exec shaded-deps-s3a-aws-sdk thrift-deps fb-deps
- yarn-extras function-registry ext-data-source query-event-hook-api impala-parent
- COMMAND ${CMAKE_SOURCE_DIR}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/fe/pom.xml b/fe/pom.xml
index 3f81c5d..d84e16d 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -24,7 +24,7 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../impala-parent/pom.xml</relativePath>
+ <relativePath>../java/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/impala-parent/CMakeLists.txt b/java/CMakeLists.txt
similarity index 93%
rename from impala-parent/CMakeLists.txt
rename to java/CMakeLists.txt
index fdd6e98..08feb30 100644
--- a/impala-parent/CMakeLists.txt
+++ b/java/CMakeLists.txt
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-add_custom_target(impala-parent ALL
+add_custom_target(java ALL DEPENDS gen-deps function-registry
COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
)
diff --git a/testdata/TableFlattener/.gitignore b/java/TableFlattener/.gitignore
similarity index 100%
rename from testdata/TableFlattener/.gitignore
rename to java/TableFlattener/.gitignore
diff --git a/testdata/TableFlattener/README b/java/TableFlattener/README
similarity index 100%
rename from testdata/TableFlattener/README
rename to java/TableFlattener/README
diff --git a/testdata/TableFlattener/pom.xml b/java/TableFlattener/pom.xml
similarity index 97%
rename from testdata/TableFlattener/pom.xml
rename to java/TableFlattener/pom.xml
index cb232d4..40a404c 100644
--- a/testdata/TableFlattener/pom.xml
+++ b/java/TableFlattener/pom.xml
@@ -23,7 +23,6 @@
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java
similarity index 100%
rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java
rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java
diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java
similarity index 100%
rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java
rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java
diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java
similarity index 100%
rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java
rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java
diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java
similarity index 100%
rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java
rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java
diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java
similarity index 100%
rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java
rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java
diff --git a/testdata/pom.xml b/java/datagenerator/pom.xml
similarity index 98%
rename from testdata/pom.xml
rename to java/datagenerator/pom.xml
index 80e0e38..03195da 100644
--- a/testdata/pom.xml
+++ b/java/datagenerator/pom.xml
@@ -24,7 +24,6 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java
similarity index 100%
rename from testdata/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java
rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java
diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java
similarity index 100%
rename from testdata/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java
rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java
diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java
similarity index 100%
rename from testdata/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java
rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java
diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
similarity index 100%
rename from testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
diff --git a/mvn-deps/executor-deps/pom.xml b/java/executor-deps/pom.xml
similarity index 96%
rename from mvn-deps/executor-deps/pom.xml
rename to java/executor-deps/pom.xml
index 1b51fa8..f9e8897 100644
--- a/mvn-deps/executor-deps/pom.xml
+++ b/java/executor-deps/pom.xml
@@ -35,7 +35,6 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
@@ -107,6 +106,12 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-minimal-s3a-aws-sdk</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -159,6 +164,12 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-minimal-hive-exec</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- IMPALA-8766: Include Knox jars on the classpath -->
diff --git a/ext-data-source/.gitignore b/java/ext-data-source/.gitignore
similarity index 100%
rename from ext-data-source/.gitignore
rename to java/ext-data-source/.gitignore
diff --git a/ext-data-source/api/pom.xml b/java/ext-data-source/api/pom.xml
similarity index 100%
rename from ext-data-source/api/pom.xml
rename to java/ext-data-source/api/pom.xml
diff --git a/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java b/java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java
similarity index 100%
rename from ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java
rename to java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java
diff --git a/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java b/java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java
similarity index 100%
rename from ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java
rename to java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java
diff --git a/ext-data-source/pom.xml b/java/ext-data-source/pom.xml
similarity index 96%
rename from ext-data-source/pom.xml
rename to java/ext-data-source/pom.xml
index b4b0045..8b4081d 100644
--- a/ext-data-source/pom.xml
+++ b/java/ext-data-source/pom.xml
@@ -23,7 +23,6 @@
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/ext-data-source/sample/pom.xml b/java/ext-data-source/sample/pom.xml
similarity index 100%
rename from ext-data-source/sample/pom.xml
rename to java/ext-data-source/sample/pom.xml
diff --git a/ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java b/java/ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java
similarity index 100%
rename from ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java
rename to java/ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java
diff --git a/ext-data-source/test/pom.xml b/java/ext-data-source/test/pom.xml
similarity index 100%
rename from ext-data-source/test/pom.xml
rename to java/ext-data-source/test/pom.xml
diff --git a/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java b/java/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java
similarity index 100%
rename from ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java
rename to java/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java
diff --git a/impala-parent/pom.xml b/java/pom.xml
similarity index 96%
rename from impala-parent/pom.xml
rename to java/pom.xml
index 3baac4d..5c8649e 100644
--- a/impala-parent/pom.xml
+++ b/java/pom.xml
@@ -219,4 +219,17 @@ under the License.
</pluginRepository>
</pluginRepositories>
+ <modules>
+ <module>datagenerator</module>
+ <module>executor-deps</module>
+ <module>ext-data-source</module>
+ <module>../fe</module>
+ <module>query-event-hook-api</module>
+ <module>shaded-deps/hive-exec</module>
+ <module>shaded-deps/s3a-aws-sdk</module>
+ <module>TableFlattener</module>
+ <module>test-hive-udfs</module>
+ <module>yarn-extras</module>
+ </modules>
+
</project>
diff --git a/query-event-hook-api/pom.xml b/java/query-event-hook-api/pom.xml
similarity index 97%
rename from query-event-hook-api/pom.xml
rename to java/query-event-hook-api/pom.xml
index 8466e0b..4e9cb99 100644
--- a/query-event-hook-api/pom.xml
+++ b/java/query-event-hook-api/pom.xml
@@ -23,7 +23,6 @@
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java b/java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java
similarity index 100%
rename from query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java
rename to java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java
diff --git a/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java b/java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java
similarity index 100%
rename from query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java
rename to java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java
diff --git a/mvn-deps/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
similarity index 98%
rename from mvn-deps/shaded-deps/hive-exec/pom.xml
rename to java/shaded-deps/hive-exec/pom.xml
index 33cb897..ea28a6b 100644
--- a/mvn-deps/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -28,7 +28,7 @@ the same dependencies
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../../impala-parent/pom.xml</relativePath>
+ <relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/mvn-deps/shaded-deps/s3a-aws-sdk/pom.xml b/java/shaded-deps/s3a-aws-sdk/pom.xml
similarity index 99%
rename from mvn-deps/shaded-deps/s3a-aws-sdk/pom.xml
rename to java/shaded-deps/s3a-aws-sdk/pom.xml
index 392ea10..aebf3d8 100644
--- a/mvn-deps/shaded-deps/s3a-aws-sdk/pom.xml
+++ b/java/shaded-deps/s3a-aws-sdk/pom.xml
@@ -26,7 +26,7 @@ though some of them might not be necessary. The exclusions are sorted alphabetic
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../../impala-parent/pom.xml</relativePath>
+ <relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/tests/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml
similarity index 98%
rename from tests/test-hive-udfs/pom.xml
rename to java/test-hive-udfs/pom.xml
index e51d598..dc3b431 100644
--- a/tests/test-hive-udfs/pom.xml
+++ b/java/test-hive-udfs/pom.xml
@@ -23,7 +23,6 @@ under the License.
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java b/java/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java b/java/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java b/java/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java b/java/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java
diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java
similarity index 100%
rename from tests/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java
rename to java/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java
diff --git a/common/yarn-extras/.gitignore b/java/yarn-extras/.gitignore
similarity index 100%
rename from common/yarn-extras/.gitignore
rename to java/yarn-extras/.gitignore
diff --git a/common/yarn-extras/README.txt b/java/yarn-extras/README.txt
similarity index 100%
rename from common/yarn-extras/README.txt
rename to java/yarn-extras/README.txt
diff --git a/common/yarn-extras/pom.xml b/java/yarn-extras/pom.xml
similarity index 98%
rename from common/yarn-extras/pom.xml
rename to java/yarn-extras/pom.xml
index 9222ef6..ce2d7f8 100644
--- a/common/yarn-extras/pom.xml
+++ b/java/yarn-extras/pom.xml
@@ -23,7 +23,6 @@
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
<version>0.1-SNAPSHOT</version>
- <relativePath>../../impala-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java
similarity index 100%
rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java
rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java
diff --git a/mvn-deps/executor-deps/CMakeLists.txt b/mvn-deps/executor-deps/CMakeLists.txt
deleted file mode 100644
index 2f80db4..0000000
--- a/mvn-deps/executor-deps/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(executor-deps ALL DEPENDS shaded-deps-hive-exec shaded-deps-s3a-aws-sdk
- COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/mvn-deps/shaded-deps/hive-exec/CMakeLists.txt b/mvn-deps/shaded-deps/hive-exec/CMakeLists.txt
deleted file mode 100644
index 7d8b6b1..0000000
--- a/mvn-deps/shaded-deps/hive-exec/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(shaded-deps-hive-exec ALL DEPENDS impala-parent
- COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt b/mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt
deleted file mode 100644
index 956f5eb..0000000
--- a/mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(shaded-deps-s3a-aws-sdk ALL DEPENDS impala-parent
- COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/query-event-hook-api/CMakeLists.txt b/query-event-hook-api/CMakeLists.txt
deleted file mode 100644
index 7352ed2..0000000
--- a/query-event-hook-api/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The dependency on yarn-extras is only added to avoid parallel downloads
-# of dependencies. For more details see IMPALA-7051, which was a similar issue.
-add_custom_target(query-event-hook-api ALL DEPENDS gen-deps impala-parent yarn-extras
- COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)
diff --git a/testdata/bin/copy-data-sources.sh b/testdata/bin/copy-data-sources.sh
index 1782aca..c218e15 100755
--- a/testdata/bin/copy-data-sources.sh
+++ b/testdata/bin/copy-data-sources.sh
@@ -28,5 +28,5 @@ setup_report_build_error
hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/data-sources/
hadoop fs -put -f \
- ${IMPALA_HOME}/ext-data-source/test/target/impala-data-source-test-*.jar \
+ ${IMPALA_HOME}/java/ext-data-source/test/target/impala-data-source-test-*.jar \
${FILESYSTEM_PREFIX}/test-warehouse/data-sources/test-data-source.jar
diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh
index def7812..438eff4 100755
--- a/testdata/bin/copy-udfs-udas.sh
+++ b/testdata/bin/copy-udfs-udas.sh
@@ -51,7 +51,7 @@ then
pushd "${IMPALA_HOME}"
"${MAKE_CMD:-make}" ${IMPALA_MAKE_FLAGS} "-j${IMPALA_BUILD_THREADS:-4}" \
TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir
- cd "${IMPALA_HOME}/tests/test-hive-udfs"
+ cd "${IMPALA_HOME}/java/test-hive-udfs"
"${IMPALA_HOME}/bin/mvn-quiet.sh" package
cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar"
# Change one of the Java files to make a new jar for testing purposes, then change it
@@ -64,9 +64,13 @@ then
bash -c "sed s/'ReplaceStringUdf'/'NewReplaceStringUdf'/g '{}'" \; \
> src/main/java/org/apache/impala/NewReplaceStringUdf.java
"${IMPALA_HOME}/bin/mvn-quiet.sh" package
+ cp target/test-hive-udfs-1.0.jar \
+ "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs-modified.jar"
find . -type f -name 'TestUpdateUdf.java' -execdir \
bash -c "sed -i s/'Text(\"New UDF\")'/'Text(\"Old UDF\")'/g '{}'" \;
rm src/main/java/org/apache/impala/NewReplaceStringUdf.java
+ # Rebuild with the original version
+ "${IMPALA_HOME}/bin/mvn-quiet.sh" package
popd
fi
@@ -91,7 +95,7 @@ ln -s "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so" \
mkdir "${UDF_TMP_DIR}/udf_test"
ln -s "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so" "${UDF_TMP_DIR}/udf_test"
ln -s "${HIVE_HOME}/lib/hive-exec-"*.jar "${UDF_TMP_DIR}/hive-exec.jar"
-ln -s "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar" \
+ln -s "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" \
"${UDF_TMP_DIR}/impala-hive-udfs.jar"
ln -s "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll" "${UDF_TMP_DIR}"
ln -s "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so" "${UDF_TMP_DIR}"
diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh
index 312bd3a..c2dfe33 100755
--- a/testdata/bin/create-load-data.sh
+++ b/testdata/bin/create-load-data.sh
@@ -479,7 +479,7 @@ function load-custom-data {
function build-and-copy-hive-udfs {
# Build the test Hive UDFs
- pushd ${IMPALA_HOME}/tests/test-hive-udfs
+ pushd ${IMPALA_HOME}/java/test-hive-udfs
${IMPALA_HOME}/bin/mvn-quiet.sh clean
${IMPALA_HOME}/bin/mvn-quiet.sh package
popd
diff --git a/testdata/bin/generate-load-nested.sh b/testdata/bin/generate-load-nested.sh
index cceea1a..b773ed0 100755
--- a/testdata/bin/generate-load-nested.sh
+++ b/testdata/bin/generate-load-nested.sh
@@ -55,7 +55,7 @@ while getopts ":n:e:i:f" OPTION; do
done
if $FLATTEN; then
- mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/TableFlattener/pom.xml" package
+ mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" package
fi
RANDOM_SCHEMA_GENERATOR=${IMPALA_HOME}/testdata/bin/random_avro_schema.py;
@@ -79,12 +79,12 @@ do
do
FILE_NAME=$(basename ${AVRO_SCHEMA_PATH})
TABLE_NAME="${FILE_NAME%.*}"
- mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/pom.xml" exec:java \
+ mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/datagenerator/pom.xml" exec:java \
-Dexec.mainClass="org.apache.impala.datagenerator.RandomNestedDataGenerator" \
-Dexec.args="${AVRO_SCHEMA_PATH} ${NUM_ELEMENTS} ${NUM_FIELDS} ${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet";
if $FLATTEN; then
- mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/TableFlattener/pom.xml" \
+ mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" \
exec:java -Dexec.mainClass=org.apache.impala.infra.tableflattener.Main \
-Dexec.arguments="file://${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet,file://${DB_DIR}/${TABLE_NAME}_flat,-sfile://${AVRO_SCHEMA_PATH}";
fi
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index 7a4336b..6e3b296 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -145,7 +145,7 @@ class TestCoordinators(CustomClusterTestSuite):
old_src_path = os.path.join(
os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar')
new_src_path = os.path.join(
- os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar')
+ os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar')
tgt_dir = get_fs_path('/test-warehouse/{0}.db'.format(db_name))
tgt_path = tgt_dir + "/tmp.jar"
diff --git a/tests/query_test/test_udfs.py b/tests/query_test/test_udfs.py
index 3d73918..4d2177d 100644
--- a/tests/query_test/test_udfs.py
+++ b/tests/query_test/test_udfs.py
@@ -517,7 +517,7 @@ class TestUdfTargeted(TestUdfBase):
old_udf = os.path.join(
os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar')
new_udf = os.path.join(
- os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar')
+ os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar')
udf_dst = get_fs_path(
'/test-warehouse/{0}.db/impala-hive-udfs.jar'.format(unique_database))
@@ -550,7 +550,7 @@ class TestUdfTargeted(TestUdfBase):
old_udf = os.path.join(
os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar')
new_udf = os.path.join(
- os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar')
+ os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar')
udf_dst = get_fs_path(
'/test-warehouse/{0}.db/impala-hive-udfs.jar'.format(unique_database))
old_function_name = "udf_update_test_create1"
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 8f1e8d3..8168cd1 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -44,7 +44,7 @@ VALID_TEST_DIRS = ['failure', 'query_test', 'stress', 'unittests', 'aux_query_te
# new dir to the list of valid test dirs above. All dirs unders tests/ must be placed
# into one of these lists, otherwise the script will throw an error. This list can be
# removed once IMPALA-4417 has been resolved.
-TEST_HELPER_DIRS = ['aux_parquet_data_load', 'test-hive-udfs', 'comparison', 'benchmark',
+TEST_HELPER_DIRS = ['aux_parquet_data_load', 'comparison', 'benchmark',
'custom_cluster', 'util', 'experiments', 'verifiers', 'common',
'performance', 'beeswax', 'aux_custom_cluster_tests',
'authorization']
[impala] 02/02: IMPALA-10198 (part 2): Add support for mvn
versions:set
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 97792c4bad526c84fbad01c98ef6b09b6c7cbcdf
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Oct 6 17:06:40 2020 -0700
IMPALA-10198 (part 2): Add support for mvn versions:set
This adds support for setting the version of Java
artifacts through "mvn versions:set". It changes
the modules to inherit the version from the parent
pom.
Previously, we used a mix of 0.1-SNAPSHOT and
1.0-SNAPSHOT. This now uses 4.0.0-SNAPSHOT across the
board. With each release, we can use "mvn versions:set"
to update the versions. The only exception is the
Hive UDF code that we build for testing. This remains
at version 1.0 to avoid test changes.
Testing:
- Ran core job
- Added build-all-flag-combinations.sh case that
does "mvn versions:set" and runs a build
Change-Id: I661b32e1e445169bac2ffe4f9474f14090031743
Reviewed-on: http://gerrit.cloudera.org:8080/16559
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/jenkins/build-all-flag-combinations.sh | 42 +++++++++++++++++++++++++++++-
docker/setup_build_context.py | 10 +++++--
fe/pom.xml | 4 +--
java/TableFlattener/pom.xml | 4 +--
java/datagenerator/pom.xml | 8 +++---
java/executor-deps/pom.xml | 2 +-
java/ext-data-source/api/pom.xml | 4 +--
java/ext-data-source/pom.xml | 4 +--
java/ext-data-source/sample/pom.xml | 4 +--
java/ext-data-source/test/pom.xml | 8 +-----
java/pom.xml | 6 ++---
java/query-event-hook-api/pom.xml | 4 +--
java/shaded-deps/hive-exec/pom.xml | 3 +--
java/shaded-deps/s3a-aws-sdk/pom.xml | 3 +--
java/test-hive-udfs/pom.xml | 3 +--
java/yarn-extras/pom.xml | 31 +++++++++++++++++++---
16 files changed, 93 insertions(+), 47 deletions(-)
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index 9209e48..87ad92a 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -88,8 +88,48 @@ for CONFIG in "${CONFIGS[@]}"; do
cat logs/mvn/mvn.log >> "${TMP_DIR}/mvn_accumulated.log"
done
+# It is useful to be able to use "mvn versions:set" to modify the jar
+# versions produced by Impala. This tests that Impala continues to build
+# after running that command.
+NEW_MAVEN_VERSION=4.0.0-$(date +%Y%m%d)
+DESCRIPTION="Options: mvn versions:set -DnewVersion=${NEW_MAVEN_VERSION}"
+
+if [[ $# == 1 && $1 == "--dryrun" ]]; then
+ echo $DESCRIPTION
+else
+ # Note: this command modifies files in the git checkout
+ pushd java
+ mvn versions:set -DnewVersion=${NEW_MAVEN_VERSION}
+ popd
+
+ if ! ./bin/clean.sh; then
+ echo "Clean failed"
+ exit 1
+ fi
+
+ echo "Building with OPTIONS: $DESCRIPTION"
+ if ! time -p ./buildall.sh -skiptests -noclean ; then
+ echo "Build failed: $DESCRIPTION"
+ FAILED="${FAILED}:${DESCRIPTION}"
+ fi
+
+ # Reset the files changed by mvn versions:set
+ git reset --hard HEAD
+
+ ccache -s
+ bin/jenkins/get_maven_statistics.sh logs/mvn/mvn.log
+
+ # Keep each maven log from each round of the build
+ cp logs/mvn/mvn.log "${TMP_DIR}/mvn.$(date +%s.%N).log"
+ # Append the maven log to the accumulated maven log
+ cat logs/mvn/mvn.log >> "${TMP_DIR}/mvn_accumulated.log"
+fi
+
# Restore the maven logs (these don't interfere with existing mvn.log)
-cp ${TMP_DIR}/mvn* logs/mvn
+# These files may not exist if this is doing a dryrun.
+if ls ${TMP_DIR}/mvn* > /dev/null 2>&1 ; then
+ cp ${TMP_DIR}/mvn* logs/mvn
+fi
if [[ "$FAILED" != "" ]]
then
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index f41a757..c1164e3 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -131,9 +131,15 @@ for jar in dep_classpath.split(":"):
symlink_file_into_dir(jar, LIB_DIR)
# Impala Coordinator jars.
-for jar in glob.glob(
- os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*-SNAPSHOT.jar")):
+num_frontend_jars = 0
+for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")):
+ # Ignore the tests jar
+ if jar.find("-tests") != -1:
+ continue
symlink_file_into_dir(jar, LIB_DIR)
+ num_frontend_jars += 1
+# There must be exactly one impala-frontend jar.
+assert num_frontend_jars == 1
# Impala Executor dependencies.
dep_classpath = file(os.path.join(IMPALA_HOME,
diff --git a/fe/pom.xml b/fe/pom.xml
index d84e16d..9bb7c7b 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -23,13 +23,11 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../java/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-frontend</artifactId>
- <version>0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Impala Query Engine Frontend</name>
diff --git a/java/TableFlattener/pom.xml b/java/TableFlattener/pom.xml
index 40a404c..53374b2 100644
--- a/java/TableFlattener/pom.xml
+++ b/java/TableFlattener/pom.xml
@@ -22,13 +22,11 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>nested-table-flattener</artifactId>
<name>Impala Nested Table Flattener</name>
- <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<build>
diff --git a/java/datagenerator/pom.xml b/java/datagenerator/pom.xml
index 03195da..b14a6e4 100644
--- a/java/datagenerator/pom.xml
+++ b/java/datagenerator/pom.xml
@@ -23,16 +23,14 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
- <artifactId>impala-testdata</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <artifactId>impala-data-generator</artifactId>
<packaging>jar</packaging>
- <name>Build some test data</name>
+ <name>Test Data Generator</name>
<dependencies>
<!-- Force json-smart dependency.
diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml
index f9e8897..4817f0a 100644
--- a/java/executor-deps/pom.xml
+++ b/java/executor-deps/pom.xml
@@ -34,7 +34,7 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/java/ext-data-source/api/pom.xml b/java/ext-data-source/api/pom.xml
index 82e6a88..6a6bd20 100644
--- a/java/ext-data-source/api/pom.xml
+++ b/java/ext-data-source/api/pom.xml
@@ -23,11 +23,9 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source-api</artifactId>
- <version>1.0-SNAPSHOT</version>
<name>Apache Impala External Data Source API</name>
<description>External Data Source API interface definition</description>
<packaging>jar</packaging>
diff --git a/java/ext-data-source/pom.xml b/java/ext-data-source/pom.xml
index 8b4081d..6d02159 100644
--- a/java/ext-data-source/pom.xml
+++ b/java/ext-data-source/pom.xml
@@ -22,13 +22,11 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
<name>Apache Impala External Data Source</name>
- <version>1.0-SNAPSHOT</version>
<packaging>pom</packaging>
<modules>
diff --git a/java/ext-data-source/sample/pom.xml b/java/ext-data-source/sample/pom.xml
index c0df755..29a88f3 100644
--- a/java/ext-data-source/sample/pom.xml
+++ b/java/ext-data-source/sample/pom.xml
@@ -23,11 +23,9 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source-sample</artifactId>
- <version>1.0-SNAPSHOT</version>
<name>Apache Impala External Data Source Sample</name>
<description>External Data Source Sample</description>
<packaging>jar</packaging>
diff --git a/java/ext-data-source/test/pom.xml b/java/ext-data-source/test/pom.xml
index c96c86a..b2a175b 100644
--- a/java/ext-data-source/test/pom.xml
+++ b/java/ext-data-source/test/pom.xml
@@ -23,20 +23,14 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source-test</artifactId>
- <version>0.1-SNAPSHOT</version>
<name>Apache Impala External Data Source Test Library</name>
<description>Test External Data Source</description>
<packaging>jar</packaging>
<url>.</url>
- <properties>
- <impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.impala</groupId>
diff --git a/java/pom.xml b/java/pom.xml
index 5c8649e..05c9a7b 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@ under the License.
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Impala Parent POM</name>
@@ -45,8 +45,8 @@ under the License.
<kite.version>${env.IMPALA_KITE_VERSION}</kite.version>
<knox.version>${env.IMPALA_KNOX_VERSION}</knox.version>
<thrift.version>0.9.3-1</thrift.version>
- <impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version>
- <impala.query.event.hook.api.version>1.0-SNAPSHOT</impala.query.event.hook.api.version>
+ <impala.extdatasrc.api.version>${project.version}</impala.extdatasrc.api.version>
+ <impala.query.event.hook.api.version>${project.version}</impala.query.event.hook.api.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<kudu.version>${env.IMPALA_KUDU_JAVA_VERSION}</kudu.version>
<commons-io.version>2.6</commons-io.version>
diff --git a/java/query-event-hook-api/pom.xml b/java/query-event-hook-api/pom.xml
index 4e9cb99..ddd64ad 100644
--- a/java/query-event-hook-api/pom.xml
+++ b/java/query-event-hook-api/pom.xml
@@ -22,13 +22,11 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>query-event-hook-api</artifactId>
<name>Apache Impala Query Event Hook API</name>
- <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<url>.</url>
diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
index ea28a6b..ce1f876 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -27,11 +27,10 @@ the same dependencies
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-minimal-hive-exec</artifactId>
<packaging>jar</packaging>
diff --git a/java/shaded-deps/s3a-aws-sdk/pom.xml b/java/shaded-deps/s3a-aws-sdk/pom.xml
index aebf3d8..665ebaf 100644
--- a/java/shaded-deps/s3a-aws-sdk/pom.xml
+++ b/java/shaded-deps/s3a-aws-sdk/pom.xml
@@ -25,11 +25,10 @@ though some of them might not be necessary. The exclusions are sorted alphabetic
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-minimal-s3a-aws-sdk</artifactId>
<packaging>jar</packaging>
diff --git a/java/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml
index dc3b431..eb63bb6 100644
--- a/java/test-hive-udfs/pom.xml
+++ b/java/test-hive-udfs/pom.xml
@@ -22,11 +22,10 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>test-hive-udfs</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
diff --git a/java/yarn-extras/pom.xml b/java/yarn-extras/pom.xml
index ce2d7f8..68ba84d 100644
--- a/java/yarn-extras/pom.xml
+++ b/java/yarn-extras/pom.xml
@@ -22,12 +22,10 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>yarn-extras</artifactId>
- <version>0.1-SNAPSHOT</version>
<name>YARN Extras</name>
<description>Extra Hadoop classes from YARN needed by Impala</description>
<packaging>jar</packaging>
@@ -45,7 +43,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>28.1-jre</version>
+ <version>${guava.version}</version>
</dependency>
<dependency>
@@ -58,6 +56,23 @@
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -77,6 +92,14 @@
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>