You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2023/07/16 13:41:02 UTC
[impala] branch master updated (0df29b410 -> 8d0ab2b68)
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 0df29b410 IMPALA-12283: Remove Hive libs from PYTHONPATH
new e62043c1f IMPALA-12287: Use old INSERT OVERWRITE TABLE syntax for Hive dataload
new a281d8eb8 IMPALA-12284: Use Maven's batch mode when building jamm
new 8d0ab2b68 IMPALA-10262: RPM/DEB Packaging Support
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 110 +++++++++++++++++++++
be/src/service/CMakeLists.txt | 4 +
bin/bootstrap_system.sh | 2 +-
bin/impala-config.sh | 3 +
bin/jenkins/build-all-flag-combinations.sh | 2 +
bin/rat_exclude_files.txt | 3 +
buildall.sh | 12 +++
docker/install_os_packages.sh | 47 +++++++--
java/CMakeLists.txt | 2 +-
package/bin/impala-env.sh | 85 ++++++++++++++++
.../bin/start-catalogd.sh | 26 +++--
package/bin/start-impalad.sh | 49 +++++++++
.../bin/start-statestored.sh | 25 +++--
.../bin/stop-catalogd.sh | 6 +-
.../add_override.sh => package/bin/stop-impalad.sh | 6 +-
.../bin/stop-statestored.sh | 6 +-
package/conf/catalogd_flags | 13 +++
package/conf/core-site.xml | 20 ++++
.../conf/fair-scheduler.xml | 4 +-
package/conf/hdfs-site.xml | 40 ++++++++
package/conf/hive-site.xml | 20 ++++
package/conf/impalad_flags | 16 +++
.../conf/llama-site.xml | 1 +
package/conf/statestore_flags | 7 ++
.../functional/functional_schema_template.sql | 16 +--
25 files changed, 479 insertions(+), 46 deletions(-)
create mode 100644 package/bin/impala-env.sh
copy testdata/bin/copy-data-sources.sh => package/bin/start-catalogd.sh (52%)
create mode 100755 package/bin/start-impalad.sh
copy testdata/bin/copy-data-sources.sh => package/bin/start-statestored.sh (55%)
copy bin/cmake_aux/add_override.sh => package/bin/stop-catalogd.sh (87%)
copy bin/cmake_aux/add_override.sh => package/bin/stop-impalad.sh (87%)
copy bin/cmake_aux/add_override.sh => package/bin/stop-statestored.sh (87%)
create mode 100644 package/conf/catalogd_flags
create mode 100644 package/conf/core-site.xml
copy fe/src/test/resources/fair-scheduler-empty.xml => package/conf/fair-scheduler.xml (55%)
create mode 100644 package/conf/hdfs-site.xml
create mode 100644 package/conf/hive-site.xml
create mode 100644 package/conf/impalad_flags
copy fe/src/test/resources/llama-site-empty.xml => package/conf/llama-site.xml (98%)
create mode 100644 package/conf/statestore_flags
[impala] 03/03: IMPALA-10262: RPM/DEB Packaging Support
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8d0ab2b684dd158f17d504e66f2b82470b83c3e1
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Sep 5 13:13:11 2022 +0800
IMPALA-10262: RPM/DEB Packaging Support
This patch bases on a previous patch contributed by Shant Hovsepian:
https://gerrit.cloudera.org/c/16612/
It adds a new option, -package, to buildall.sh for building a package
for the current OS type (e.g. CentOS/Ubuntu). You can also use
"make/ninja package" to build the package. Scripts for launching the
services and the required configuration files are also added.
Tests:
- Built on Ubuntu 18.04/20.04 and CentOS 7 using
./buildall.sh -noclean -skiptests -release -package
- Deployed the RPM package on a CDP cluster. Verifed the scripts.
- Deployed the DEB package on a docker container. Verified the scripts.
Change-Id: I64419fd400fe8d233dac016b6306157fe9461d82
Reviewed-on: http://gerrit.cloudera.org:8080/18939
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
CMakeLists.txt | 110 +++++++++++++++++++++++++++++
be/src/service/CMakeLists.txt | 4 ++
bin/bootstrap_system.sh | 2 +-
bin/impala-config.sh | 3 +
bin/jenkins/build-all-flag-combinations.sh | 2 +
bin/rat_exclude_files.txt | 3 +
buildall.sh | 12 ++++
docker/install_os_packages.sh | 47 ++++++++++--
package/bin/impala-env.sh | 85 ++++++++++++++++++++++
package/bin/start-catalogd.sh | 38 ++++++++++
package/bin/start-impalad.sh | 49 +++++++++++++
package/bin/start-statestored.sh | 37 ++++++++++
package/bin/stop-catalogd.sh | 23 ++++++
package/bin/stop-impalad.sh | 23 ++++++
package/bin/stop-statestored.sh | 23 ++++++
package/conf/catalogd_flags | 13 ++++
package/conf/core-site.xml | 20 ++++++
package/conf/fair-scheduler.xml | 6 ++
package/conf/hdfs-site.xml | 40 +++++++++++
package/conf/hive-site.xml | 20 ++++++
package/conf/impalad_flags | 16 +++++
package/conf/llama-site.xml | 3 +
package/conf/statestore_flags | 7 ++
23 files changed, 579 insertions(+), 7 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a0f7a210f..7559ebfc6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,6 +24,8 @@ set(NO_TESTS 1)
# initialized.
project(Impala)
+option(BUILD_PACKAGES "Build deployment packages")
+
include(cmake_modules/kudu_cmake_fns.txt)
if (NOT DEFINED BUILD_SHARED_LIBS)
@@ -485,6 +487,15 @@ else()
set_property(GLOBAL PROPERTY RULE_LAUNCH_CUSTOM ${JUNITXML_WRAPPER})
endif()
+cmake_host_system_information(RESULT OS_DISTRIB_ID QUERY DISTRIB_ID)
+cmake_host_system_information(RESULT OS_DISTRIB_VERSION_ID QUERY DISTRIB_VERSION_ID)
+message(STATUS "DISTRIB_ID: ${OS_DISTRIB_ID}")
+message(STATUS "DISTRIB_VERSION_ID: ${OS_DISTRIB_VERSION_ID}")
+
+## installation path
+set(CMAKE_INSTALL_PREFIX "/opt")
+set(IMPALA_INSTALLDIR "impala" CACHE INTERNAL "")
+
# compile these subdirs using their own CMakeLists.txt
add_subdirectory(common/function-registry)
add_subdirectory(common/thrift)
@@ -495,6 +506,42 @@ add_subdirectory(docker)
add_subdirectory(java)
add_subdirectory(shell)
+install(DIRECTORY "www/" DESTINATION ${IMPALA_INSTALLDIR}/www)
+install(FILES fe/target/impala-frontend-$ENV{IMPALA_VERSION}.jar
+ DESTINATION ${IMPALA_INSTALLDIR}/jar)
+
+set(IMPALA_GCC_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/gcc-$ENV{IMPALA_GCC_VERSION})
+FILE(GLOB gcc_lib ${IMPALA_GCC_HOME}/lib64/libgcc_s.so.1*)
+install(FILES ${gcc_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+FILE(GLOB cpp_lib ${IMPALA_GCC_HOME}/lib64/libstdc++.so.6*)
+install(FILES ${cpp_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+set(KUDU_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$ENV{IMPALA_KUDU_VERSION}/release)
+# The parent folder is lib64 on centos/redhat, while on ubuntu it's lib.
+FILE(GLOB kudu_lib ${KUDU_HOME}/lib*/libkudu_client.so*)
+install(FILES ${kudu_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+FILE(GLOB hadoop_lib $ENV{HADOOP_LIB_DIR}/native/libhadoop.so*)
+install(FILES ${hadoop_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+install(DIRECTORY fe/target/dependency/ DESTINATION ${IMPALA_INSTALLDIR}/jar
+ FILES_MATCHING PATTERN "*.jar")
+
+string(TOLOWER ${OS_DISTRIB_ID} OS_DISTRIB_ID)
+if (${OS_DISTRIB_ID} STREQUAL "rhel" AND ${OS_DISTRIB_VERSION_ID} VERSION_GREATER_EQUAL 8)
+ # TODO: remove this after resolving IMPALA-12255
+ message(STATUS "Not packaging impala-shell on redhat8+ due to ambiguous python shebang")
+else()
+ install(DIRECTORY shell/build/impala-shell-$ENV{IMPALA_VERSION}/
+ DESTINATION ${IMPALA_INSTALLDIR}/shell
+ USE_SOURCE_PERMISSIONS PATTERN "*.pyc" EXCLUDE)
+endif()
+
+install(DIRECTORY package/bin/ DESTINATION ${IMPALA_INSTALLDIR}/bin
+ USE_SOURCE_PERMISSIONS FILES_MATCHING PATTERN "*.sh")
+install(DIRECTORY package/conf/ DESTINATION ${IMPALA_INSTALLDIR}/conf)
+
# Build target for all generated files which most backend code depends on
add_custom_target(gen-deps ALL DEPENDS thrift-deps proto-deps fb-deps
kudu-util-proto-deps kudu-rpc-proto-deps kudu-security-proto-deps gen_ir_descriptions)
@@ -543,3 +590,66 @@ if (DUMP_INCLUDE_PATHS)
endif(DUMP_INCLUDE_PATHS)
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libstdc++")
+
+# packaging
+if (BUILD_PACKAGES)
+
+ if (${OS_DISTRIB_ID} MATCHES "(rhel|centos)")
+ set(CPACK_GENERATOR "RPM")
+ elseif (${OS_DISTRIB_ID} MATCHES "ubuntu")
+ set(CPACK_GENERATOR "DEB")
+ else()
+ message(WARNING "OS is ${OS_DISTRIB_ID}. No packages will be generated.")
+ endif()
+
+ if (CPACK_GENERATOR)
+ message(STATUS "Packaging enabled: ${CPACK_GENERATOR}")
+ set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME})
+ set(CPACK_PACKAGE_VENDOR "Apache")
+ set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An open source massively parallel processing SQL query engine.")
+ set(CPACK_PACKAGE_CONTACT "dev@impala.apache.org")
+ set(CPACK_PACKAGE_HOMEPAGE_URL "https://impala.apache.org")
+ set(CPACK_PACKAGE_VERSION "$ENV{IMPALA_VERSION}")
+ set(CPACK_RPM_PACKAGE_LICENSE "ASL-2.0")
+ # Set a meaningful package name, e.g. Impala-4.3.0-SNAPSHOT_hive-3.1.3000.7.2.18.0-41-x86_64.el7
+ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}")
+ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}_hive-$ENV{IMPALA_HIVE_VERSION}")
+ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}-${CMAKE_SYSTEM_PROCESSOR}")
+ if (${CPACK_GENERATOR} MATCHES "RPM")
+ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.el${OS_DISTRIB_VERSION_ID}")
+ elseif (${CPACK_GENERATOR} MATCHES "DEB")
+ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.ubuntu-${OS_DISTRIB_VERSION_ID}")
+ endif()
+ message(STATUS "Package name: ${CPACK_PACKAGE_FILE_NAME}")
+
+ if ($ENV{STRIP_DEPLOYMENT_IMPALAD})
+ set(CPACK_STRIP_FILES ${IMPALA_INSTALLDIR}/bin/impalad)
+ message("Binaries in the package will be stripped")
+ endif()
+
+ set(CPACK_PACKAGING_INSTALL_PREFIX "/opt")
+ set(CPACK_PACKAGE_INSTALL_DIRECTORY "impala")
+
+ execute_process(
+ COMMAND bash -c "${CMAKE_SOURCE_DIR}/docker/install_os_packages.sh --dry-run | tail -n1"
+ OUTPUT_VARIABLE PKG_LIST
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+ if ("${PKG_LIST}" STREQUAL "")
+ message(FATAL_ERROR "Package list is empty: '${PKG_LIST}'")
+ else()
+ message(STATUS "Get required package list: '${PKG_LIST}'")
+ endif()
+
+ if (${CPACK_GENERATOR} MATCHES "RPM")
+ set(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
+ set(CPACK_RPM_PACKAGE_RELOCATABLE TRUE)
+ set(CPACK_RPM_PACKAGE_REQUIRES ${PKG_LIST})
+ else()
+ set(CPACK_DEBIAN_PACKAGE_DEPENDS ${PKG_LIST})
+ endif()
+ endif()
+
+include(CPack)
+
+endif()
diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt
index 538772bf3..f8b1b3b5e 100644
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -126,6 +126,10 @@ target_link_libraries(unifiedbetests
${JAVA_JSIG_LIBRARY} ${UNIFIED_TEST_LINK_LIBS})
ADD_DEPENDENCIES(unified-be-test-executable unifiedbetests)
+install(FILES ${STATESTORED_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
+install(FILES ${CATALOGD_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
+install(TARGETS impalad DESTINATION ${IMPALA_INSTALLDIR}/bin)
+
# Exception to unified be tests: Custom main() due to leak
ADD_BE_TEST(session-expiry-test session-expiry-test.cc) # TODO: this leaks thrift server
ADD_UNIFIED_BE_LSAN_TEST(hs2-util-test "StitchNullsTest.*:PrintTColumnValueTest.*")
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index 965d434ea..15f14f32b 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -260,7 +260,7 @@ ubuntu sudo update-java-alternatives -s \
redhat sudo yum install -y file gawk gcc gcc-c++ git krb5-devel krb5-server \
krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \
cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \
- postgresql postgresql-server \
+ postgresql postgresql-server rpm-build \
wget vim-common nscd cmake zlib-devel \
procps psmisc lsof openssh-server python3-devel python3-setuptools \
net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 8359791ac..b10541439 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -1008,6 +1008,9 @@ CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH"
# A marker in the environment to prove that we really did source this file
export IMPALA_CONFIG_SOURCED=1
+# Whether to strip the impalad binary when generating deployment package
+export STRIP_DEPLOYMENT_IMPALAD=true
+
echo "IMPALA_VERSION = $IMPALA_VERSION"
echo "IMPALA_HOME = $IMPALA_HOME"
echo "HADOOP_HOME = $HADOOP_HOME"
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index d10f92f06..e54ab396b 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -39,6 +39,7 @@ CONFIGS=(
# Test gcc builds with and without -so:
"-skiptests -noclean"
"-skiptests -noclean -release"
+ "-skiptests -noclean -release -package"
"-skiptests -noclean -release -so -ninja"
# clang sanitizer builds:
"-skiptests -noclean -asan"
@@ -46,6 +47,7 @@ CONFIGS=(
"-skiptests -noclean -ubsan -so -ninja"
# USE_APACHE_HIVE=true build:
"-skiptests -noclean -use_apache_hive"
+ "-skiptests -noclean -use_apache_hive -package"
)
FAILED=""
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 3cef6ca20..8b4ca15f5 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -254,3 +254,6 @@ docs/images/howto_show_histogram.png
docs/images/howto_static_server_pools_config.png
docs/images/impala_arch.jpeg
docs/images/support_send_diagnostic_data.png
+
+# Files of deployment configuration
+package/conf/*
diff --git a/buildall.sh b/buildall.sh
index b6d2da415..82835852b 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -69,6 +69,7 @@ BUILD_ASAN=0
BUILD_FE_ONLY=0
BUILD_TESTS=1
GEN_CMAKE_ONLY=0
+GEN_PACKAGE=0
BUILD_RELEASE_AND_DEBUG=0
BUILD_TIDY=0
BUILD_UBSAN=0
@@ -204,6 +205,9 @@ do
-cmake_only)
GEN_CMAKE_ONLY=1
;;
+ -package)
+ GEN_PACKAGE=1
+ ;;
-help|*)
echo "buildall.sh - Builds Impala and runs all tests."
echo "[-noclean] : Omits cleaning all packages before building. Will not kill"\
@@ -258,6 +262,7 @@ do
echo "[-fe_only] : Build just the frontend"
echo "[-ninja] : Use ninja instead of make"
echo "[-cmake_only] : Generate makefiles only, instead of doing a full build"
+ echo "[-package] : Generate a package for deployment."
echo "-----------------------------------------------------------------------------
Examples of common tasks:
@@ -537,6 +542,9 @@ generate_cmake_files() {
echo "CACHELINESIZE_AARCH64:$CACHELINESIZE_AARCH64"
CMAKE_ARGS+=(-DCACHELINESIZE_AARCH64=$CACHELINESIZE_AARCH64)
fi
+ if [[ "$GEN_PACKAGE" -eq 1 ]]; then
+ CMAKE_ARGS+=(-DBUILD_PACKAGES=ON)
+ fi
cmake . ${CMAKE_ARGS[@]}
}
@@ -660,6 +668,10 @@ else
build_all_components $CMAKE_BUILD_TYPE 1
fi
+if [[ "$GEN_PACKAGE" -eq 1 ]]; then
+ ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} package
+fi
+
if [[ $NEED_MINICLUSTER -eq 1 ]]; then
reconfigure_test_cluster
fi
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index 0c863a045..8fbeef793 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -25,11 +25,36 @@ set -euo pipefail
INSTALL_DEBUG_TOOLS=false
JAVA_VERSION=8
+DRY_RUN=false
+PKG_LIST=""
+NON_PKG_NAMES=(apt-get yum install update)
function print_usage {
echo "install_os_packages.sh - Helper script to install OS dependencies"
echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc"
echo "[--java <version>] : Use specified Java version rather than the default Java 8."
+ echo "[--dry-run] : Print the list of packages to install."
+}
+
+# Wraps the passed in command to either execute it (DRY_RUN=false) or just use it
+# to update PKG_LIST.
+function wrap {
+ if $DRY_RUN; then
+ for arg in $@; do
+ if [[ "${NON_PKG_NAMES[@]}" =~ "$arg" ]]; then
+ continue
+ elif [[ "$arg" == "-"* ]]; then
+ # Ignores command options
+ continue
+ elif [[ "$PKG_LIST" != "" ]]; then
+ PKG_LIST="$PKG_LIST,$arg"
+ else
+ PKG_LIST="$arg"
+ fi
+ done
+ else
+ "$@"
+ fi
}
while [ -n "$*" ]
@@ -42,6 +67,9 @@ do
JAVA_VERSION="${2-}"
shift;
;;
+ --dry-run)
+ DRY_RUN=true
+ ;;
--help|*)
print_usage
exit 1
@@ -52,6 +80,7 @@ done
echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
echo "JAVA_VERSION=${JAVA_VERSION}"
+echo "DRY_RUN=${DRY_RUN}"
# This can get more detailed if there are specific steps
# for specific versions, but at the moment the distribution
@@ -85,8 +114,8 @@ fi
# Optionally install extra debug tools.
if [[ $DISTRIBUTION == Ubuntu ]]; then
export DEBIAN_FRONTEND=noninteractive
- apt-get update
- apt-get install -y \
+ wrap apt-get update
+ wrap apt-get install -y \
hostname \
krb5-user \
language-pack-en \
@@ -97,7 +126,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
tzdata
if $INSTALL_DEBUG_TOOLS ; then
echo "Installing extra debug tools"
- apt-get install -y \
+ wrap apt-get install -y \
curl \
dnsutils \
iproute2 \
@@ -111,7 +140,7 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
if [[ $JAVA_VERSION == 8 ]]; then
JAVA_VERSION=1.8.0
fi
- yum install -y --disableplugin=subscription-manager \
+ wrap yum install -y --disableplugin=subscription-manager \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
hostname \
@@ -125,14 +154,14 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
# Install the appropriate language packs. Redhat/Centos 7 come
# with en_US.utf8, so there is no need to install anything.
if ! grep 'release 7\.' /etc/redhat-release; then
- yum install -y --disableplugin=subscription-manager \
+ wrap yum install -y --disableplugin=subscription-manager \
glibc-langpack-en \
langpacks-en
fi
if $INSTALL_DEBUG_TOOLS ; then
echo "Installing extra debug tools"
- yum install -y --disableplugin=subscription-manager \
+ wrap yum install -y --disableplugin=subscription-manager \
bind-utils \
curl \
iproute \
@@ -145,6 +174,12 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
fi
fi
+if $DRY_RUN; then
+ echo "The following packages would be installed:"
+ echo "$PKG_LIST"
+ exit 0
+fi
+
# Verify en_US.utf8 is present
if ! locale -a | grep en_US.utf8 ; then
echo "ERROR: en_US.utf8 locale is not present."
diff --git a/package/bin/impala-env.sh b/package/bin/impala-env.sh
new file mode 100644
index 000000000..3655ac6cf
--- /dev/null
+++ b/package/bin/impala-env.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [[ -z "$JAVA_HOME" ]]; then
+ echo "JAVA_HOME not set!"
+ exit 1
+fi
+
+echo "Using JAVA_HOME: $JAVA_HOME"
+LIB_JVM_DIR=$(dirname $(find $JAVA_HOME -type f -name libjvm.so))
+LIB_JSIG_DIR=$(dirname $(find $JAVA_HOME -type f -name libjsig.so))
+
+export LC_ALL=en_US.utf8
+export LD_LIBRARY_PATH="/opt/impala/lib/:$LIB_JVM_DIR:$LIB_JSIG_DIR"
+export CLASSPATH="/opt/impala/conf:/opt/impala/jar/*"
+
+#TODO: Add graceful shutdown for impalads
+function stop_process {
+ name=$1
+ pid_file="/tmp/${name}.pid"
+ if [[ -f $pid_file ]]; then
+ PID=$(cat $pid_file)
+ if ps $PID | grep $name; then
+ echo "Killing $name with PID=$PID"
+ kill $PID
+ rm $pid_file
+ echo "Killed $name"
+ else
+ rm $pid_file
+ echo "Already stopped: $name is not running with PID=$PID. Removed stale $pid_file"
+ fi
+ else
+ echo "PID file $pid_file not found!"
+ fi
+}
+
+function wait_for_ready {
+ name=$1
+ port=$2
+ pid=$3
+
+ NUM_WAIT_ITERATIONS=20
+ i=0
+ while [[ $i -lt $NUM_WAIT_ITERATIONS ]]; do
+ if ! ps $pid | grep $name > /dev/null 2>&1; then
+ echo "$name with PID $pid doesn't exist"
+ break
+ fi
+ STATUS=$(curl -s http://localhost:$port/healthz)
+ if [[ $? != 0 ]]; then
+ echo "Waiting for $name. Port $port not ready."
+ elif [[ "$STATUS" != "OK" ]]; then
+ echo "Waiting for $name to be ready"
+ else
+ echo "$name is ready"
+ break
+ fi
+ sleep 2
+ i=$((i+1))
+ done
+ if [[ "$STATUS" == "OK" ]]; then
+ echo "Launched $name with PID $pid"
+ elif [[ $i -eq $NUM_WAIT_ITERATIONS ]]; then
+ echo "Timed out waiting for $name to be ready. Check logs for more details."
+ else
+ echo "Failed to launch $name"
+ exit 1
+ fi
+}
diff --git a/package/bin/start-catalogd.sh b/package/bin/start-catalogd.sh
new file mode 100755
index 000000000..9b472fe60
--- /dev/null
+++ b/package/bin/start-catalogd.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch catalogd. Required JAVA_HOME being set.
+# Edit conf/catalogd_flags to set the correct hostname and state_store_host.
+# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster.
+# Example usage:
+# export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+# bin/start-catalogd.sh
+# To launch catalogd using another username (e.g. "impala"):
+# sudo -E -u impala bin/start-catalogd.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+$IMPALA_HOME/bin/catalogd --flagfile=$IMPALA_HOME/conf/catalogd_flags &
+PID=$!
+echo $PID > /tmp/catalogd.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready catalogd 25020 $PID
diff --git a/package/bin/start-impalad.sh b/package/bin/start-impalad.sh
new file mode 100755
index 000000000..89ccbbd21
--- /dev/null
+++ b/package/bin/start-impalad.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch impalad. Required JAVA_HOME being set.
+# Edit conf/impalad_flags to set the correct hostnames.
+# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster.
+# Example usage:
+# export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+# bin/start-impalad.sh
+# To launch impalad using another username (e.g. "impala"):
+# sudo -E -u impala bin/start-impalad.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+if [[ -n "$HADOOP_HOME" ]]; then
+ echo "Using HADOOP_HOME: $HADOOP_HOME"
+ export HADOOP_LIB_DIR="${HADOOP_HOME}/lib"
+ export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${HADOOP_LIB_DIR}/native/"
+ echo "Using hadoop native libs in ${HADOOP_LIB_DIR}/native/"
+else
+ export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${IMPALA_HOME}/lib"
+ echo "HADOOP_HOME not set. Using hadoop native libs in ${IMPALA_HOME}/lib"
+fi
+
+$IMPALA_HOME/bin/impalad --flagfile=$IMPALA_HOME/conf/impalad_flags &
+PID=$!
+echo $PID > /tmp/impalad.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready impalad 25000 $PID
diff --git a/package/bin/start-statestored.sh b/package/bin/start-statestored.sh
new file mode 100755
index 000000000..e3c78cd34
--- /dev/null
+++ b/package/bin/start-statestored.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch statestore. Required JAVA_HOME being set.
+# Edit conf/statestore_flags to set a correct hostname.
+# Example usage:
+# export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+# bin/start-statestored.sh
+# To launch statestore using another username (e.g. "impala"):
+# sudo -E -u impala bin/start-statestored.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+$IMPALA_HOME/bin/statestored --flagfile=$IMPALA_HOME/conf/statestore_flags &
+PID=$!
+echo $PID > /tmp/statestored.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready statestored 25010 $PID
diff --git a/package/bin/stop-catalogd.sh b/package/bin/stop-catalogd.sh
new file mode 100755
index 000000000..8649ed143
--- /dev/null
+++ b/package/bin/stop-catalogd.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process catalogd
diff --git a/package/bin/stop-impalad.sh b/package/bin/stop-impalad.sh
new file mode 100755
index 000000000..4e323e889
--- /dev/null
+++ b/package/bin/stop-impalad.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process impalad
diff --git a/package/bin/stop-statestored.sh b/package/bin/stop-statestored.sh
new file mode 100755
index 000000000..e5aa9578d
--- /dev/null
+++ b/package/bin/stop-statestored.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process statestored
diff --git a/package/conf/catalogd_flags b/package/conf/catalogd_flags
new file mode 100644
index 000000000..7b94816a2
--- /dev/null
+++ b/package/conf/catalogd_flags
@@ -0,0 +1,13 @@
+-hostname=localhost
+-state_store_host=localhost
+#-kudu_master_hosts=localhost
+
+-log_dir=/var/log/catalogd
+-log_filename=catalogd
+-state_store_port=24000
+-minidump_path=/var/log/impala-minidumps
+-webserver_doc_root=/opt/impala
+-catalog_topic_mode=minimal
+-hms_event_polling_interval_s=0
+-v=1
+-max_log_size=200
diff --git a/package/conf/core-site.xml b/package/conf/core-site.xml
new file mode 100644
index 000000000..2303cdf5f
--- /dev/null
+++ b/package/conf/core-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://localhost:8020</value>
+ </property>
+ <property>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>ipc.client.connect.max.retries</name>
+ <value>50</value>
+ </property>
+ <property>
+ <name>fs.trash.interval</name>
+ <value>1</value>
+ </property>
+</configuration>
diff --git a/package/conf/fair-scheduler.xml b/package/conf/fair-scheduler.xml
new file mode 100644
index 000000000..0edf7f080
--- /dev/null
+++ b/package/conf/fair-scheduler.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<allocations>
+ <queue name="root">
+ <queue name="default"/>
+ </queue>
+</allocations>
diff --git a/package/conf/hdfs-site.xml b/package/conf/hdfs-site.xml
new file mode 100644
index 000000000..02468a478
--- /dev/null
+++ b/package/conf/hdfs-site.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+ <property>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value>localhost:8022</value>
+ </property>
+ <property>
+ <name>dfs.namenode.http-address</name>
+ <value>localhost:20101</value>
+ </property>
+ <property>
+ <name>dfs.replication</name>
+ <value>3</value>
+ </property>
+ <property>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
+ </property>
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/run/hdfs-sockets/dn</value>
+ </property>
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.client.read.shortcircuit.streams.cache.size</name>
+ <value>4096</value>
+ </property>
+ <property>
+ <name>dfs.client.read.shortcircuit.skip.checksum</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dfs.client.file-block-storage-locations.timeout.millis</name>
+ <value>10000</value>
+ </property>
+</configuration>
diff --git a/package/conf/hive-site.xml b/package/conf/hive-site.xml
new file mode 100644
index 000000000..ec614f2b5
--- /dev/null
+++ b/package/conf/hive-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://localhost:9083</value>
+ </property>
+ <property>
+ <name>hive.support.concurrency</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hive.metastore.client.socket.timeout</name>
+ <value>3600</value>
+ </property>
+ <property>
+ <name>hive.metastore.connect.retries</name>
+ <value>5</value>
+ </property>
+</configuration>
diff --git a/package/conf/impalad_flags b/package/conf/impalad_flags
new file mode 100644
index 000000000..4c2dc3a92
--- /dev/null
+++ b/package/conf/impalad_flags
@@ -0,0 +1,16 @@
+-hostname=localhost
+-state_store_host=localhost
+-catalog_service_host=localhost
+#-kudu_master_hosts=localhost
+
+-mem_limit=80%
+-use_local_catalog=true
+-log_dir=/var/log/impalad
+-log_filename=impalad
+-minidump_path=/var/log/impala-minidumps
+-local_library_dir=/var/lib/impala/udfs
+-fair_scheduler_allocation_path=/opt/impala/conf/fair-scheduler.xml
+-llama_site_path=/opt/impala/conf/llama-site.xml
+-webserver_doc_root=/opt/impala
+-v=1
+-max_log_size=200
diff --git a/package/conf/llama-site.xml b/package/conf/llama-site.xml
new file mode 100644
index 000000000..fefdb93ca
--- /dev/null
+++ b/package/conf/llama-site.xml
@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration/>
diff --git a/package/conf/statestore_flags b/package/conf/statestore_flags
new file mode 100644
index 000000000..9e53d3a70
--- /dev/null
+++ b/package/conf/statestore_flags
@@ -0,0 +1,7 @@
+-hostname=localhost
+-log_dir=/var/log/statestore
+-log_filename=statestored
+-minidump_path=/var/log/impala-minidumps
+-webserver_doc_root=/opt/impala
+-v=1
+-max_log_size=200
[impala] 01/03: IMPALA-12287: Use old INSERT OVERWRITE TABLE syntax for Hive dataload
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e62043c1fa16acf4e6cff4320b5ac2545c055d27
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Fri Jul 14 12:06:13 2023 -0700
IMPALA-12287: Use old INSERT OVERWRITE TABLE syntax for Hive dataload
In dataload, we have some Hive statements that use the "INSERT OVERWRITE"
syntax rather than the "INSERT OVERWRITE TABLE" syntax. Older versions
of Hive do not support this syntax. In order to keep the dataload code
as compatible as possible, this switches the "INSERT OVERWRITE"
statements to "INSERT OVERWRITE TABLE".
Testing:
- Ran a core job
Change-Id: I455641280166c39dcc42fb4187f728df8148cc70
Reviewed-on: http://gerrit.cloudera.org:8080/20198
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../datasets/functional/functional_schema_template.sql | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 31ccd7d17..3a5e5b076 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -3781,7 +3781,7 @@ map_timestamp_key MAP<TIMESTAMP, STRING>
map_date_key MAP<DATE, STRING>
---- DEPENDENT_LOAD_HIVE
-- It would be nice to insert NULLs, but I couldn't find a way in Hive.
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES
(1,
array(1, 2, NULL),
array(array(1, 2, NULL), array(3)),
@@ -3842,7 +3842,7 @@ map_timestamp_key MAP<TIMESTAMP, STRING>
map_date_key MAP<DATE, STRING>
struct_contains_map STRUCT<m: MAP<INT, STRING>, s: STRING>
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES
(1,
map(true, "true", if(false, false, NULL), "null"),
map(-1Y, "one", if(false, 1Y, NULL), "null"),
@@ -3871,7 +3871,7 @@ id INT
map_int_int MAP<INT,INT>
map_char3_char5 MAP<CHAR(3),CHAR(5)>
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES
(1, map(10, 100, 11, 110, 12, 120), map(cast("aaa" as char(3)), cast("aaaaa" as char(5)))),
(2, map(20, 200, 21, 210, 22, 220), map(cast("aab" as char(3)), cast("aaaab" as char(5)))),
(3, map(30, 300, 31, 310, 32, 320), map(cast("aac" as char(3)), cast("aaaac" as char(5)))),
@@ -3897,7 +3897,7 @@ arr_contains_nested_struct ARRAY<STRUCT<inner_struct: STRUCT<str: STRING, l: INT
struct_contains_nested_arr STRUCT<arr: ARRAY<ARRAY<DATE>>, i: INT>
all_mix MAP<INT, STRUCT<big: STRUCT<arr: ARRAY<STRUCT<inner_arr: ARRAY<ARRAY<INT>>, m: TIMESTAMP>>, n: INT>, small: STRUCT<str: STRING, i: INT>>>
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES
(
1,
named_struct("arr", array(1, 2, 3, 4, NULL, NULL, 5)),
@@ -4143,7 +4143,7 @@ date_string_col string
string_col string
timestamp_col timestamp
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} select * from functional.alltypesagg;
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} select * from functional.alltypesagg;
---- TABLE_PROPERTIES
parquet.writer.version=v2
parquet.compression=UNCOMPRESSED
@@ -4169,7 +4169,7 @@ date_string_col string
string_col string
timestamp_col timestamp
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} select * from functional.alltypesagg;
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} select * from functional.alltypesagg;
---- TABLE_PROPERTIES
parquet.writer.version=v2
parquet.compression=SNAPPY
@@ -4186,7 +4186,7 @@ int_map map<string, int>
int_map_array array<map<string, int>>
nested_struct struct<a: int, b: array<int>, c: struct<d: array<array<struct<e: int, f: string>>>>, g: map<string, struct<h: struct<i: array<double>>>>>
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} select * from functional_parquet.complextypestbl;
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} select * from functional_parquet.complextypestbl;
---- TABLE_PROPERTIES
parquet.writer.version=v2
parquet.compression=UNCOMPRESSED
@@ -4203,7 +4203,7 @@ int_map map<string, int>
int_map_array array<map<string, int>>
nested_struct struct<a: int, b: array<int>, c: struct<d: array<array<struct<e: int, f: string>>>>, g: map<string, struct<h: struct<i: array<double>>>>>
---- DEPENDENT_LOAD_HIVE
-INSERT OVERWRITE {db_name}{db_suffix}.{table_name} select * from functional_parquet.complextypestbl;
+INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} select * from functional_parquet.complextypestbl;
---- TABLE_PROPERTIES
parquet.writer.version=v2
parquet.compression=SNAPPY
[impala] 02/03: IMPALA-12284: Use Maven's batch mode when building jamm
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a281d8eb8e41f453f4edfec9f4fd1f25e994fe39
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Thu Jul 13 19:24:34 2023 -0700
IMPALA-12284: Use Maven's batch mode when building jamm
This adds the --batch-mode flag to the maven invocation
the builds jamm. That disables some of the download progress
output, reducing the total size of the output.
Testing:
- Ran a build locally
Change-Id: I1634240b191168b13cf3be7c9266e21a746844b1
Reviewed-on: http://gerrit.cloudera.org:8080/20196
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
java/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt
index 89313e1f4..aa3c6de12 100644
--- a/java/CMakeLists.txt
+++ b/java/CMakeLists.txt
@@ -27,7 +27,7 @@ ExternalProject_Add(jamm
PATCH_COMMAND sed -i.bak s/0.4.0-SNAPSHOT/0.4.0-IMPALA/ pom.xml
CONFIGURE_COMMAND sed s:JAVA_HOME:$ENV{JAVA_HOME}:g ${CMAKE_CURRENT_SOURCE_DIR}/toolchains.xml.tmpl > toolchains.xml
BUILD_IN_SOURCE true
- BUILD_COMMAND mvn --toolchains toolchains.xml install -Dmaven.test.skip=true -DskipTests
+ BUILD_COMMAND mvn --batch-mode --toolchains toolchains.xml install -Dmaven.test.skip=true -DskipTests
INSTALL_COMMAND ""
)