You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/09/09 05:21:53 UTC
[impala] branch master updated (f598b2ad6 -> 5aff120c4)
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from f598b2ad6 IMPALA-10610: Support multiple file formats in a single Iceberg Table
new 1644cf629 IMPALA-9448: Use Ozone TDE in minicluster
new 5aff120c4 IMPALA-11540: Add logs for ALTER_TABLE events that trigger slow metadata reload
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/impala-config.sh | 1 +
.../impala/catalog/events/MetastoreEvents.java | 20 ++++++++++++++++++++
testdata/bin/run-all.sh | 1 -
.../bin/{kill-ranger-server.sh => setup-dfs-keys.sh} | 10 ++++++++--
testdata/bin/setup-hdfs-env.sh | 11 ++---------
testdata/cluster/admin | 15 ++++++++++++---
6 files changed, 43 insertions(+), 15 deletions(-)
copy testdata/bin/{kill-ranger-server.sh => setup-dfs-keys.sh} (79%)
[impala] 01/02: IMPALA-9448: Use Ozone TDE in minicluster
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1644cf629bd2ca47f54e77756a266e8f193777be
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Aug 30 15:26:57 2022 -0700
IMPALA-9448: Use Ozone TDE in minicluster
Enables Ozone's Transparent Data Encryption when creating a bucket in
the minicluster. Requires `-format` to turn on encryption in an existing
minicluster.
Tested by running e2e and custom cluster tests with Ozone.
Change-Id: Icd82967cdab7b222e5876936f39e5198ba9773ea
Reviewed-on: http://gerrit.cloudera.org:8080/18931
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 1 +
testdata/bin/run-all.sh | 1 -
testdata/bin/setup-dfs-keys.sh | 30 ++++++++++++++++++++++++++++++
testdata/bin/setup-hdfs-env.sh | 11 ++---------
testdata/cluster/admin | 15 ++++++++++++---
5 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 736f3fc8c..4aec2d408 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -657,6 +657,7 @@ elif [ "${TARGET_FILESYSTEM}" = "hdfs" ]; then
export HDFS_ERASURECODE_PATH="/test-warehouse"
fi
elif [ "${TARGET_FILESYSTEM}" = "ozone" ]; then
+ export USE_OZONE_ENCRYPTION=${USE_OZONE_ENCRYPTION-true}
export OZONE_VOLUME="impala"
export OZONE_BUCKET="base"
export DEFAULT_FS="o3fs://${OZONE_BUCKET}.${OZONE_VOLUME}.${INTERNAL_LISTEN_HOST}:9862"
diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh
index 801552144..dda171c23 100755
--- a/testdata/bin/run-all.sh
+++ b/testdata/bin/run-all.sh
@@ -81,7 +81,6 @@ else
# - We don't yet have a good way to start YARN using a different defaultFS. Moreoever
# we currently don't run hive queries against Isilon for testing.
# - LLAMA is avoided because we cannot start YARN.
- # - KMS is used for encryption testing, which is not available on remote storage.
# - Hive needs YARN, and we don't run Hive queries.
# Impala can also run on a local file system without additional services.
# TODO: Figure out how to start YARN, LLAMA and Hive with a different defaultFs.
diff --git a/testdata/bin/setup-dfs-keys.sh b/testdata/bin/setup-dfs-keys.sh
new file mode 100755
index 000000000..aeb55f8a5
--- /dev/null
+++ b/testdata/bin/setup-dfs-keys.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -euo pipefail
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
+
+# Create encryption keys for HDFS encryption tests. Keys are stored by the KMS.
+EXISTING_KEYS=$(hadoop key list)
+for KEY in $*; do
+ if ! grep $KEY <<< $EXISTING_KEYS &>/dev/null; then
+ hadoop key create $KEY -f
+ fi
+done
diff --git a/testdata/bin/setup-hdfs-env.sh b/testdata/bin/setup-hdfs-env.sh
index 51ab42ea6..2b81a49e3 100755
--- a/testdata/bin/setup-hdfs-env.sh
+++ b/testdata/bin/setup-hdfs-env.sh
@@ -46,15 +46,8 @@ fi
# against a remote cluster, rather than the local mini-cluster (i.e., when REMOTE_LOAD
# is true. See: IMPALA-4344)
-if [[ $TARGET_FILESYSTEM == hdfs && -z "$REMOTE_LOAD" ]]; then # Otherwise assume KMS isn't setup.
- # Create encryption keys for HDFS encryption tests. Keys are stored by the KMS.
- EXISTING_KEYS=$(hadoop key list)
- for KEY in testkey{1,2}; do
- if grep $KEY <<< $EXISTING_KEYS &>/dev/null; then
- hadoop key delete $KEY -f
- fi
- hadoop key create $KEY
- done
+if [[ -z "$REMOTE_LOAD" ]]; then # Otherwise assume KMS isn't setup.
+ ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey{1,2}
fi
if [[ -n "${REMOTE_LOAD:-}" ]]; then
diff --git a/testdata/cluster/admin b/testdata/cluster/admin
index 4bce41712..a8ed6ef16 100755
--- a/testdata/cluster/admin
+++ b/testdata/cluster/admin
@@ -57,7 +57,7 @@ if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then
# The check above indicates that the regular mini-cluster is in use.
SUPPORTED_SERVICES=(hdfs kms yarn)
elif [[ "$TARGET_FILESYSTEM" == "ozone" ]]; then
- SUPPORTED_SERVICES=(ozone)
+ SUPPORTED_SERVICES=(kms ozone)
else
# Either a remote distributed file system or a local non-distributed file system is
# in use. Currently the only service that is expected to work is Kudu, though in theory
@@ -327,8 +327,17 @@ function start_cluster {
fi
if [[ "${TARGET_FILESYSTEM}" = "ozone" ]]; then
- ozone sh volume create /${OZONE_VOLUME} || true
- ozone sh bucket create /${OZONE_VOLUME}/${OZONE_BUCKET} || true
+ local bucketkey=''
+ if $USE_OZONE_ENCRYPTION; then
+ echo "Ozone encryption enabled for ${OZONE_VOLUME}/${OZONE_BUCKET}"
+
+ # Encryption is done at the bucket level, so ensure the keys are available first.
+ ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey
+ bucketkey='--bucketkey testkey'
+ fi
+
+ ozone sh volume create ${OZONE_VOLUME} || true
+ ozone sh bucket create ${bucketkey} ${OZONE_VOLUME}/${OZONE_BUCKET} || true
fi
return $?
[impala] 02/02: IMPALA-11540: Add logs for ALTER_TABLE events that trigger slow metadata reload
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5aff120c40475de35bf45703b3cb953b9ab970f1
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Aug 29 08:14:37 2022 +0800
IMPALA-11540: Add logs for ALTER_TABLE events that trigger slow metadata reload
ALTER_TABLE event could trigger full refresh on the table, which is
time consuming and can lead to event-processor lagging behind. We need
logs to show the difference between the before and after hmsTables so we
can further distinguish the cases that actully don't need a full table
refresh, e.g. changing the owner, changing tblproperties, etc.
Tests:
- Attach the process using IntelliJ to pause the metadata loading for a
while. So the metadata reloading of ALTER_TABLE event can exceed the
threshold. Manually verified the logs.
Change-Id: Ibf344e6b423f88c9635ca8d61d53385b88ba4dce
Reviewed-on: http://gerrit.cloudera.org:8080/18926
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Quanlong Huang <hu...@gmail.com>
---
.../impala/catalog/events/MetastoreEvents.java | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 2c8e6c98d..4bfb2c8a4 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -64,6 +64,7 @@ import org.apache.impala.catalog.TableNotFoundException;
import org.apache.impala.catalog.TableNotLoadedException;
import org.apache.impala.catalog.TableWriteId;
import org.apache.impala.common.Metrics;
+import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.Reference;
import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.hive.common.MutableValidWriteIdList;
@@ -641,6 +642,17 @@ public class MetastoreEvents {
LOG.trace(formatString, formatArgs);
}
+ /**
+ * Similar to infoLog excepts logs at warn level
+ */
+ protected void warnLog(String logFormattedStr, Object... args) {
+ if (!LOG.isWarnEnabled()) return;
+ String formatString =
+ new StringBuilder(LOG_FORMAT_EVENT_ID_TYPE).append(logFormattedStr).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ LOG.warn(formatString, formatArgs);
+ }
+
/**
* Search for a inverse event (for example drop_table is a inverse event for
* create_table) for this event from a given list of notificationEvents starting for
@@ -1432,6 +1444,7 @@ public class MetastoreEvents {
// refresh eg. this could be due to as simple as adding a new parameter or a
// full blown adding or changing column type
// rename is already handled above
+ long startNs = System.nanoTime();
if (!reloadTableFromCatalog("ALTER_TABLE", false)) {
if (wasEventSyncTurnedOn()) {
// we received this alter table event on a non-existing table. We also
@@ -1448,6 +1461,13 @@ public class MetastoreEvents {
+ "the event processing state", getFullyQualifiedTblName()));
}
}
+ long durationNs = System.nanoTime() - startNs;
+ // Log event details for those triggered slow reload.
+ if (durationNs > HdfsTable.LOADING_WARNING_TIME_NS) {
+ warnLog("Slow event processing. Duration: {}. TableBefore: {}. " +
+ "TableAfter: {}", PrintUtils.printTimeNs(durationNs),
+ tableBefore_.toString(), tableAfter_.toString());
+ }
}
/**