You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by db...@apache.org on 2025/01/10 13:31:31 UTC
(impala) branch master updated (c518d3c81 -> 46115e9a8)
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from c518d3c81 IMPALA-13501: Clean up uncommitted Iceberg files after validation check failure
new 2a03499c6 IMPALA-13458: Fix installing curl on Red Hat variants for dockerised tests
new a45a7a374 IMPALA-13339: [DOCS] Documentation for COPY TESTCASE statements
new 46115e9a8 IMPALA-13126: Obtain table read lock in EP to process partitioned event
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docker/install_os_packages.sh | 6 +-
docs/impala.ditamap | 1 +
docs/topics/impala_copy_testcase.xml | 82 ++++++++++++++++++++++
.../impala/catalog/events/MetastoreEvents.java | 78 ++++++++++++--------
tests/custom_cluster/test_events_custom_configs.py | 25 +++++++
5 files changed, 160 insertions(+), 32 deletions(-)
create mode 100644 docs/topics/impala_copy_testcase.xml
(impala) 02/03: IMPALA-13339: [DOCS] Documentation for COPY TESTCASE statements
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a45a7a3745874a8d595a6092a4ac7a7008b85fc3
Author: m-sanjana19 <ma...@gmail.com>
AuthorDate: Tue Dec 31 11:44:48 2024 +0530
IMPALA-13339: [DOCS] Documentation for COPY TESTCASE statements
Documents the COPY TESTCASE statements used to extract and
share query metadata for debugging.
Change-Id: I4d3c96c5b0ca0723ea02a8b3fb72abcd31ef52fa
Reviewed-on: http://gerrit.cloudera.org:8080/22284
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
docs/impala.ditamap | 1 +
docs/topics/impala_copy_testcase.xml | 82 ++++++++++++++++++++++++++++++++++++
2 files changed, 83 insertions(+)
diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index 077828ec8..bfa125c3e 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -132,6 +132,7 @@ under the License.
<topicref href="topics/impala_langref_sql.xml">
<topicref href="topics/impala_ddl.xml"/>
<topicref href="topics/impala_dml.xml"/>
+ <topicref href="topics/impala_copy_testcase.xml"/>
<topicref href="topics/impala_alter_database.xml"/>
<topicref href="topics/impala_alter_table.xml"/>
<topicref href="topics/impala_alter_view.xml"/>
diff --git a/docs/topics/impala_copy_testcase.xml b/docs/topics/impala_copy_testcase.xml
new file mode 100644
index 000000000..2621deb4a
--- /dev/null
+++ b/docs/topics/impala_copy_testcase.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="impala_copy_testcase" rev="4.3.0 IMPALA-11901">
+ <title>Copy Testcase Statements</title>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="SQL"/>
+ <data name="Category" value="Copy Testcase"/>
+ <data name="Category" value="Data Analysts"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Tables"/>
+ <data name="Category" value="ETL"/>
+ <data name="Category" value="Ingest"/>
+ </metadata>
+ </prolog>
+ <conbody>
+ <p>The <codeph>COPY TESTCASE</codeph> statement allows users to export table or view metadata
+ involved in a query, allowing developers to replay the metadata in a local cluster for testing
+ and debugging purposes.</p>
+ </conbody>
+ <concept id="concept_erz_l3j_wdc">
+ <title>Exporting Metadata to a File</title>
+ <conbody>
+ <p>Use the following syntax to dump
+ metadata:<codeblock id="codeblock_bzf_g3j_wdc">COPY TESTCASE TO <hdfs/s3 dirpath> <query stmt></codeblock></p>
+ <p><b>Example:</b><codeblock id="codeblock_e4t_p3j_wdc">COPY TESTCASE TO '/tmp' SELECT * FROM functional_parquet.alltypes;
+</codeblock></p>
+ <p><b>Output:</b><codeblock id="codeblock_u44_r3j_wdc">
++--------------------------------------------------------------------------------------+
+| Test case data output path |
++--------------------------------------------------------------------------------------+
+| hdfs://localhost:20500/tmp/impala-testcase-data-f41f7b14-dfc8-408b-ac3b-ef49fc3e0a83 |
++--------------------------------------------------------------------------------------+
+</codeblock></p>
+ </conbody>
+ </concept>
+ <concept id="concept_d1v_3sp_wdc">
+ <title>Loading Metadata into a Target Cluster</title>
+ <conbody>
+ <p>To load the previously exported metadata, use the following
+ syntax:<codeblock id="codeblock_hkg_ksp_wdc">COPY TESTCASE FROM <hdfs/s3 testcase file path></codeblock></p>
+ <p><b>Example:</b><codeblock id="codeblock_hrq_lsp_wdc">COPY TESTCASE FROM '/tmp/impala-testcase-data-f41f7b14-dfc8-408b-ac3b-ef49fc3e0a83';</codeblock></p>
+ <p><b>Output:</b><codeblock id="codeblock_oqg_nsp_wdc">
++----------------------------------------------------------------------------------------------------------------+
+| summary |
++----------------------------------------------------------------------------------------------------------------+
+| Testcase generated using Impala version 4.5.0-SNAPSHOT. 1 db(s), 1 table(s) and 0 view(s) imported for query: |
+| |
+| SELECT * FROM functional_parquet.alltypes |
++----------------------------------------------------------------------------------------------------------------+
+</codeblock></p>
+ </conbody>
+ </concept>
+ <concept id="concept_tvd_psp_wdc">
+ <title>Using the Imported Metadata</title>
+ <conbody>
+ <p>To use the imported metadata in query planning, enable the planner testcase
+ mode:<codeblock id="codeblock_rzj_rsp_wdc">SET PLANNER_TESTCASE_MODE=true;</codeblock></p>
+ <p>You can now interact with the metadata using commands
+ like:<codeblock id="codeblock_g1r_ssp_wdc">SHOW COLUMN STATS functional_parquet.alltypes;</codeblock></p>
+ </conbody>
+ </concept>
+</concept>
\ No newline at end of file
(impala) 01/03: IMPALA-13458: Fix installing curl on Red Hat variants for dockerised tests
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2a03499c61b91f60129340c50d7e63f9c6263ef8
Author: Laszlo Gaal <la...@cloudera.com>
AuthorDate: Tue Oct 15 21:42:50 2024 +0200
IMPALA-13458: Fix installing curl on Red Hat variants for dockerised tests
Red Hat 8 and 9 as well as their variants (e.g. Rocky Linux) preinstall
the curl-minimal package as a prerequisite for their package manager.
Unfortunately this conflicts with the installation of the full-blown
curl package when the Impala daemon Docker images are built during a
dockerised test run. The failure is caused by the two packages having
slightly different version numbers.
Fix this the same way as in bootstrap_system.sh: add the --allowerasing
flag to the yum command line to let yum/DNF substitute the full curl
version for the preinstalled curl-minimal package.
Tested by executing dockerised tests on Rocky Linux 9.2
Change-Id: I30fa0f13a77ef2a939a1b754014a78c171443c71
Reviewed-on: http://gerrit.cloudera.org:8080/21944
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
docker/install_os_packages.sh | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index c8c419aba..4422fd579 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -187,7 +187,11 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
if [[ $INSTALL_DEBUG_TOOLS == full ]]; then
echo "Installing full debug tools"
- wrap yum install -y --disableplugin=subscription-manager \
+ # Redhat 8 and 9 come with curl-minimal preinstalled, which conflicts with the
+ # full curl package. Add --allowerasing to the yum command to allow package
+ # replacement.
+
+ wrap yum install -y --disableplugin=subscription-manager --allowerasing \
bind-utils \
curl \
iproute \
(impala) 03/03: IMPALA-13126: Obtain table read lock in EP to process partitioned event
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 46115e9a8ec1c6016cad7a1f3af7f7c260c170e3
Author: Sai Hemanth Gantasala <sa...@cloudera.com>
AuthorDate: Mon Aug 12 14:40:39 2024 -0700
IMPALA-13126: Obtain table read lock in EP to process partitioned event
For a partition-level event, isOlderEvent() in catalogD needs to check
whether the corresponding partition is reloaded after the event. This
should be done after holding the table read lock. Otherwise,
EventProcessor could hit ConcurrentModificationException error when
there are concurrent DDLs/DMLs modifying the partition list.
note: Created IMPALA-13650 for a cleaner solution to clear the inflight
events list for partitioned table events.
Testing:
- Added a end-to-end stress test to verify the above scenario
Change-Id: I26933f98556736f66df986f9440ebb64be395bc1
Reviewed-on: http://gerrit.cloudera.org:8080/21663
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../impala/catalog/events/MetastoreEvents.java | 78 +++++++++++++---------
tests/custom_cluster/test_events_custom_configs.py | 25 +++++++
2 files changed, 72 insertions(+), 31 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index c2612a878..43ec31549 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -1297,19 +1297,29 @@ public class MetastoreEvents {
return true;
}
// Always check the lastRefreshEventId on the table first for table level refresh
- if (tbl.getLastRefreshEventId() > getEventId() || (partitionEventObj != null &&
- catalog_.isPartitionLoadedAfterEvent(dbName_, tblName_,
- partitionEventObj, getEventId()))) {
- metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
- .inc(getNumberOfEvents());
- String messageStr = partitionEventObj == null ? "Skipping the event since the" +
- " table " + dbName_+ "." + tblName_ + " has last refresh id as " +
- tbl.getLastRefreshEventId() + ". Comparing it with current event " +
- getEventId() + ". " : "";
- infoLog("{}Incremented events skipped counter to {}", messageStr,
- metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
- .getCount());
- return true;
+ boolean canSkip = tbl.getLastRefreshEventId() >= getEventId();
+ try {
+ if (!canSkip && partitionEventObj != null) {
+ tbl.takeReadLock();
+ canSkip = catalog_.isPartitionLoadedAfterEvent(dbName_, tblName_,
+ partitionEventObj, getEventId());
+ }
+ if (canSkip) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(getNumberOfEvents());
+ String messageStr = partitionEventObj == null ? "Skipping the event since " +
+ "the table " + dbName_ + "." + tblName_ + " has last refresh id as " +
+ tbl.getLastRefreshEventId() + ". Comparing it with current event " +
+ getEventId() + ". " : "";
+ infoLog("{}Incremented events skipped counter to {}", messageStr,
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ return true;
+ }
+ } finally {
+ if (tbl.isReadLockedByCurrentThread()) {
+ tbl.releaseReadLock();
+ }
}
} catch (CatalogException e) {
debugLog("ignoring exception while checking if it is an older event "
@@ -1805,12 +1815,6 @@ public class MetastoreEvents {
return;
}
- if (isOlderEvent(null)) {
- infoLog("Not processing the alter table event {} as it is an older event",
- getEventId());
- return;
- }
-
// Determine whether this is an event which we have already seen or if it is a new
// event
if (isSelfEvent()) {
@@ -1825,6 +1829,12 @@ public class MetastoreEvents {
+ "which can be ignored.");
return;
}
+
+ if (isOlderEvent(null)) {
+ infoLog("Not processing the alter table event {} as it is an older event",
+ getEventId());
+ return;
+ }
skipFileMetadataReload_ = !isTruncateOp_ && canSkipFileMetadataReload(tableBefore_,
tableAfter_);
long startNs = System.nanoTime();
@@ -2640,12 +2650,6 @@ public class MetastoreEvents {
return;
}
- if (isOlderEvent(partitionBefore_)) {
- infoLog("Not processing the alter partition event {} as it is an older event",
- getEventId());
- return;
- }
-
// Ignore the event if this is a trivial event. See javadoc for
// isTrivialAlterPartitionEvent() for examples.
if (canBeSkipped()) {
@@ -2654,6 +2658,12 @@ public class MetastoreEvents {
+ "parameters which can be ignored.");
return;
}
+
+ if (isOlderEvent(partitionBefore_)) {
+ infoLog("Not processing the alter partition event {} as it is an older event",
+ getEventId());
+ return;
+ }
// Reload the whole table if it's a transactional table or materialized view.
// Materialized views are treated as a special case because it's possible to
// receive partition event on MVs, but they are regular views in Impala. That
@@ -3133,13 +3143,19 @@ public class MetastoreEvents {
org.apache.impala.catalog.Table tbl = catalog_.getTableNoThrow(dbName_, tblName_);
if (tbl == null || tbl instanceof IncompleteTable) { return false; }
// Always check the lastRefreshEventId on the table first for table level refresh
- if (tbl.getLastRefreshEventId() >= getEventId()
- || (reloadPartition_ != null
- && catalog_.isPartitionLoadedAfterEvent(
- dbName_, tblName_, reloadPartition_, getEventId()))) {
- return true;
+ boolean canSkip = tbl.getLastRefreshEventId() >= getEventId();
+ try {
+ if (!canSkip && reloadPartition_ != null) {
+ tbl.takeReadLock();
+ canSkip = catalog_.isPartitionLoadedAfterEvent(dbName_, tblName_,
+ reloadPartition_, getEventId());
+ }
+ return canSkip;
+ } finally {
+ if (tbl.isReadLockedByCurrentThread()) {
+ tbl.releaseReadLock();
+ }
}
- return false;
}
/**
diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py
index b23861622..1deaf24c8 100644
--- a/tests/custom_cluster/test_events_custom_configs.py
+++ b/tests/custom_cluster/test_events_custom_configs.py
@@ -1357,6 +1357,31 @@ class TestEventProcessingCustomConfigs(TestEventProcessingCustomConfigsBase):
EventProcessorUtils.wait_for_event_processing(self)
assert EventProcessorUtils.get_event_processor_status() == "ACTIVE"
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ catalogd_args="--enable_reload_events=true "
+ "--invalidate_metadata_on_event_processing_failure=false")
+ def test_reload_events_modified_concurrently(self):
+ """IMPALA-13126: This test verifies that the event processor successfully consumes or
+ ignores the RELOAD event triggered by refresh operation on a partitioned table if the
+ partitions are modified concurrently on the table."""
+ tbl = "scale_db.num_partitions_1234_blocks_per_partition_1"
+ refresh_stmt = "refresh {} partition(j=0)".format(tbl)
+ for _ in range(32):
+ self.client.execute_async(refresh_stmt)
+ for _ in range(100):
+ self.client.execute(
+ "alter table {} add if not exists partition(j=-1)".format(tbl))
+ self.client.execute(
+ "alter table {} drop partition(j=-1)".format(tbl))
+
+ try:
+ EventProcessorUtils.wait_for_event_processing(self, 1000) # bigger timeout required
+ assert EventProcessorUtils.get_event_processor_status() == "ACTIVE"
+ finally:
+ # Make sure the table doesn't change after this test
+ self.execute_query("alter table {} drop if exists partition(j=-1)".format(tbl))
+
@SkipIfFS.hive
class TestEventProcessingWithImpala(TestEventProcessingCustomConfigsBase):