You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by wz...@apache.org on 2024/05/10 07:24:33 UTC
(impala) branch master updated (09d2f10f4 -> 3cbb3be5f)
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 09d2f10f4 IMPALA-13040: Add waiting mechanism in UpdateFilterFromRemote
new 68f8a6a1d IMPALA-12607: Bump the GBN and fetch events specific to the db/table from the metastore
new 3cbb3be5f IMPALA-13018: Block push down of conjuncts with implicit casting on base columns for jdbc tables
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/impala-config.sh | 22 +-
.../org/apache/impala/compat/MetastoreShim.java | 6 +
.../org/apache/impala/compat/MetastoreShim.java | 20 ++
.../catalog/events/MetastoreEventsProcessor.java | 118 +++++++--
.../apache/impala/planner/DataSourceScanNode.java | 32 ++-
.../events/MetastoreEventsProcessorTest.java | 76 ++++++
.../CatalogHmsSyncToLatestEventIdTest.java | 2 +-
java/shaded-deps/hive-exec/pom.xml | 2 +
.../queries/PlannerTest/constant-folding.test | 4 +-
.../queries/PlannerTest/data-source-tables.test | 269 ++++++++++++++++++++-
.../queries/PlannerTest/small-query-opt.test | 2 +-
tests/query_test/test_tpcds_queries.py | 5 +-
12 files changed, 500 insertions(+), 58 deletions(-)
(impala) 01/02: IMPALA-12607: Bump the GBN and fetch events specific to the db/table from the metastore
Posted by wz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 68f8a6a1df0d2da91baa87b8b6699ddbc495b88e
Author: Sai Hemanth Gantasala <sa...@cloudera.com>
AuthorDate: Thu Feb 1 12:34:00 2024 -0800
IMPALA-12607: Bump the GBN and fetch events specific to the db/table
from the metastore
Bump the GBN to 49623641 to leverage HIVE-27499, so that Impala can
directly fetch the latest events specific to the db/table from the
metastore, instead of fetching the events from metastore and then
filtering in the cache matching the DbName/TableName.
Implementation Details:
Currently when a DDL/DML is performed in Impala, we fetch all the
events from metastore based on current eventId and then filter them in
Impala which can be a bottleneck if the events count is huge. This can
be optimized by including db name and/or table name in the notification
event request object and then filter by event type in impala. This can
provide performance boost on tables that generate a lot of events.
Note:
Also included ShowUtils class in hive-minimal-exec jar as it is
required in the current build version
Testing:
1) Did some tests in local cluster
2) Added a test case in MetaStoreEventsProcessorTest
Change-Id: I6aecd5108b31c24e6e2c6f9fba6d4d44a3b00729
Reviewed-on: http://gerrit.cloudera.org:8080/20979
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 22 ++--
.../org/apache/impala/compat/MetastoreShim.java | 6 ++
.../org/apache/impala/compat/MetastoreShim.java | 20 ++++
.../catalog/events/MetastoreEventsProcessor.java | 118 +++++++++++++++++----
.../events/MetastoreEventsProcessorTest.java | 76 +++++++++++++
.../CatalogHmsSyncToLatestEventIdTest.java | 2 +-
java/shaded-deps/hive-exec/pom.xml | 2 +
7 files changed, 213 insertions(+), 33 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index af391a3d7..4bcb364d3 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -255,19 +255,19 @@ fi
: ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com}
export IMPALA_TOOLCHAIN_HOST
-export CDP_BUILD_NUMBER=45689292
+export CDP_BUILD_NUMBER=49623641
export CDP_MAVEN_REPOSITORY=\
"https://${IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${CDP_BUILD_NUMBER}/maven"
-export CDP_AVRO_JAVA_VERSION=1.8.2.7.2.18.0-369
-export CDP_HADOOP_VERSION=3.1.1.7.2.18.0-369
-export CDP_HBASE_VERSION=2.4.17.7.2.18.0-369
-export CDP_HIVE_VERSION=3.1.3000.7.2.18.0-369
-export CDP_ICEBERG_VERSION=1.3.1.7.2.18.0-369
-export CDP_KNOX_VERSION=1.3.0.7.2.18.0-369
-export CDP_OZONE_VERSION=1.3.0.7.2.18.0-369
-export CDP_PARQUET_VERSION=1.10.99.7.2.18.0-369
-export CDP_RANGER_VERSION=2.4.0.7.2.18.0-369
-export CDP_TEZ_VERSION=0.9.1.7.2.18.0-369
+export CDP_AVRO_JAVA_VERSION=1.8.2.7.3.0.0-128
+export CDP_HADOOP_VERSION=3.1.1.7.3.0.0-128
+export CDP_HBASE_VERSION=2.4.17.7.3.0.0-128
+export CDP_HIVE_VERSION=3.1.3000.7.3.0.0-128
+export CDP_ICEBERG_VERSION=1.3.1.7.3.0.0-128
+export CDP_KNOX_VERSION=2.0.0.7.3.0.0-128
+export CDP_OZONE_VERSION=1.3.0.7.3.0.0-128
+export CDP_PARQUET_VERSION=1.12.3.7.3.0.0-128
+export CDP_RANGER_VERSION=2.4.0.7.3.0.0-128
+export CDP_TEZ_VERSION=0.9.1.7.3.0.0-128
# Ref: https://infra.apache.org/release-download-pages.html#closer
: ${APACHE_MIRROR:="https://www.apache.org/dyn/closer.cgi"}
diff --git a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index c34d2d8af..98c96f1d2 100644
--- a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -74,6 +74,7 @@ import org.apache.impala.catalog.Hive3MetastoreShimBase;
import org.apache.impala.catalog.MetaStoreClientPool;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEvent;
+import org.apache.impala.catalog.events.MetastoreEventsProcessor.MetaDataFilter;
import org.apache.impala.catalog.events.MetastoreNotificationException;
import org.apache.impala.catalog.events.SelfEventContext;
import org.apache.impala.catalog.local.MetaProvider.PartitionMetadata;
@@ -980,4 +981,9 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
// IMetaStoreClient.getDataConnector().
return null;
}
+
+ public static void setNotificationEventRequestWithFilter(
+ NotificationEventRequest eventRequest, MetaDataFilter metaDataFilter) {
+ // noop for non-existent fields of NotificationEventRequest
+ }
}
diff --git a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index f5354c0a4..03feafe22 100644
--- a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -33,6 +33,7 @@ import com.google.common.collect.Iterables;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -107,6 +108,7 @@ import org.apache.impala.catalog.TableWriteId;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEvent;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType;
import org.apache.impala.catalog.events.MetastoreEventsProcessor;
+import org.apache.impala.catalog.events.MetastoreEventsProcessor.MetaDataFilter;
import org.apache.impala.catalog.events.MetastoreNotificationException;
import org.apache.impala.catalog.events.MetastoreNotificationNeedsInvalidateException;
import org.apache.impala.catalog.events.SelfEventContext;
@@ -1082,4 +1084,22 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
}
return null;
}
+
+ public static void setNotificationEventRequestWithFilter(
+ NotificationEventRequest eventRequest, MetaDataFilter metaDataFilter) {
+ if (metaDataFilter != null) {
+ if (metaDataFilter.getCatName() != null &&
+ !metaDataFilter.getCatName().isEmpty()) {
+ eventRequest.setCatName(metaDataFilter.getCatName());
+ }
+ if (metaDataFilter.getDbName() != null &&
+ !metaDataFilter.getDbName().isEmpty()) {
+ eventRequest.setDbName(metaDataFilter.getDbName());
+ }
+ if (metaDataFilter.getTableName() != null &&
+ !metaDataFilter.getTableName().isEmpty()) {
+ eventRequest.setTableNames(Arrays.asList(metaDataFilter.getTableName()));
+ }
+ }
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
index 30673e835..0d0c858b7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
@@ -66,6 +66,7 @@ import org.apache.impala.service.CatalogOpExecutor;
import org.apache.impala.thrift.TEventBatchProgressInfo;
import org.apache.impala.thrift.TEventProcessorMetrics;
import org.apache.impala.thrift.TEventProcessorMetricsSummaryResponse;
+import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.MetaStoreUtil;
import org.apache.impala.util.NoOpEventSequence;
import org.apache.impala.util.ThreadNameAnnotator;
@@ -296,7 +297,9 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
eventType.equals(notificationEvent.getEventType())
&& catName.equalsIgnoreCase(notificationEvent.getCatName())
&& dbName.equalsIgnoreCase(notificationEvent.getDbName());
- return getNextMetastoreEventsInBatches(catalog, eventId, filter, eventType);
+ MetaDataFilter metaDataFilter = new MetaDataFilter(filter, catName, dbName);
+ return getNextMetastoreEventsWithFilterInBatches(catalog, eventId, metaDataFilter,
+ EVENTS_BATCH_SIZE_PER_RPC, eventType);
}
public static List<NotificationEvent> getNextMetastoreEventsInBatchesForTable(
@@ -318,7 +321,8 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
&& catName.equalsIgnoreCase(notificationEvent.getCatName())
&& dbName.equalsIgnoreCase(notificationEvent.getDbName())
&& tblName.equalsIgnoreCase(notificationEvent.getTableName());
- return getNextMetastoreEventsInBatches(catalog, eventId, filter,
+ MetaDataFilter metaDataFilter = new MetaDataFilter(filter, catName, dbName, tblName);
+ return getNextMetastoreEventsWithFilterInBatches(catalog, eventId, metaDataFilter,
EVENTS_BATCH_SIZE_PER_RPC, eventType);
}
@@ -329,27 +333,37 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
EVENTS_BATCH_SIZE_PER_RPC, eventTypes);
}
- /**
- * Gets the next list of {@link NotificationEvent} from Hive Metastore which are
- * greater than the given eventId and filtered according to the provided filter.
- * @param catalog The CatalogServiceCatalog used to get the metastore client
- * @param eventId The eventId after which the events are needed.
- * @param filter The {@link NotificationFilter} used to filter the list of fetched
- * events. Note that this is a client side filter not a server side
- * filter. Unfortunately, HMS doesn't provide a similar mechanism to
- * do server side filtering.
- * @param eventsBatchSize the batch size for fetching the events from metastore.
- * @return List of {@link NotificationEvent} which are all greater than eventId and
- * satisfy the given filter.
- * @throws MetastoreNotificationFetchException in case of RPC errors to metastore.
- */
@VisibleForTesting
public static List<NotificationEvent> getNextMetastoreEventsInBatches(
CatalogServiceCatalog catalog, long eventId, NotificationFilter filter,
int eventsBatchSize, String... eventTypes)
throws MetastoreNotificationFetchException {
+ MetaDataFilter metaDataFilter = new MetaDataFilter(filter);
+ return getNextMetastoreEventsWithFilterInBatches(catalog, eventId, metaDataFilter,
+ eventsBatchSize, eventTypes);
+ }
+
+ /**
+ * Gets the next list of {@link NotificationEvent} from Hive Metastore which are
+ * greater than the given eventId and filtered according to the provided filter.
+ * @param catalog The CatalogServiceCatalog used to get the metastore client
+ * @param eventId The eventId after which the events are needed.
+ * @param metaDataFilter The {@link MetaDataFilter} used to filter the list of
+ * fetched events based on catName/dbName/tableName and then filter
+ * by required event types. Note that this is a server side filter.
+ * @param eventsBatchSize the batch size for fetching the events from metastore.
+ * @return List of {@link NotificationEvent} which are all greater than eventId and
+ * satisfy the given filter.
+ * @throws MetastoreNotificationFetchException in case of RPC errors to metastore.
+ */
+ @VisibleForTesting
+ public static List<NotificationEvent> getNextMetastoreEventsWithFilterInBatches(
+ CatalogServiceCatalog catalog, long eventId, MetaDataFilter metaDataFilter,
+ int eventsBatchSize, String... eventTypes)
+ throws MetastoreNotificationFetchException {
Preconditions.checkArgument(eventsBatchSize > 0);
List<NotificationEvent> result = new ArrayList<>();
+ NotificationFilter filter = metaDataFilter.getNotificationFilter();
try (MetaStoreClient msc = catalog.getMetaStoreClient()) {
long toEventId = msc.getHiveClient().getCurrentNotificationEventId()
.getEventId();
@@ -376,6 +390,9 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
NotificationEventRequest eventRequest = new NotificationEventRequest();
eventRequest.setMaxEvents(batchSize);
eventRequest.setLastEvent(currentEventId);
+ // Need to set table/dbnames in the request according to the filter
+ MetastoreShim.setNotificationEventRequestWithFilter(eventRequest,
+ metaDataFilter);
NotificationEventResponse notificationEventResponse =
MetastoreShim.getNextNotification(msc.getHiveClient(), eventRequest,
eventTypeSkipList);
@@ -432,8 +449,18 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
String annotation = String.format("sync table %s to latest HMS event id",
tbl.getFullName());
try(ThreadNameAnnotator tna = new ThreadNameAnnotator(annotation)) {
- List<NotificationEvent> events = getNextMetastoreEventsInBatches(catalog,
- lastEventId, getTableNotificationEventFilter(tbl));
+ MetaDataFilter metaDataFilter;
+ // For ACID tables, events may include commit_txn and abort_txn which doesn't have
+ // db_name and table_name. So it makes sense to fetch all the events and filter
+ // them in catalogD.
+ if (AcidUtils.isTransactionalTable(tbl.getMetaStoreTable().getParameters())) {
+ metaDataFilter = new MetaDataFilter(getTableNotificationEventFilter(tbl));
+ } else {
+ metaDataFilter = new MetaDataFilter(getTableNotificationEventFilter(tbl),
+ MetastoreShim.getDefaultCatalogName(), tbl.getDb().getName(), tbl.getName());
+ }
+ List<NotificationEvent> events = getNextMetastoreEventsWithFilterInBatches(catalog,
+ lastEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC);
if (events.isEmpty()) {
LOG.debug("table {} synced till event id {}. No new HMS events to process from "
@@ -494,8 +521,11 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
String annotation = String.format("sync db %s to latest HMS event id", db.getName());
try(ThreadNameAnnotator tna = new ThreadNameAnnotator(annotation)) {
- List<NotificationEvent> events = getNextMetastoreEventsInBatches(catalog,
- lastEventId, getDbNotificationEventFilter(db));
+ MetaDataFilter metaDataFilter = new MetaDataFilter(
+ getDbNotificationEventFilter(db), MetastoreShim.getDefaultCatalogName(),
+ db.getName());
+ List<NotificationEvent> events = getNextMetastoreEventsWithFilterInBatches(catalog,
+ lastEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC);
if (events.isEmpty()) {
LOG.debug("db {} already synced till event id: {}, no new hms events from "
@@ -550,7 +580,8 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
This filter is used when syncing db to the latest HMS event id. The
filter accepts all events except table related ones
*/
- private static NotificationFilter getDbNotificationEventFilter(Db db) {
+ @VisibleForTesting
+ public static NotificationFilter getDbNotificationEventFilter(Db db) {
NotificationFilter filter = new NotificationFilter() {
@Override
public boolean accept(NotificationEvent event) {
@@ -976,6 +1007,7 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
NotificationEventRequest eventRequest = new NotificationEventRequest();
eventRequest.setLastEvent(eventId);
eventRequest.setMaxEvents(batchSize);
+ // Currently filter is always null. No need to set table/db in the request object
NotificationEventResponse response =
MetastoreShim.getNextNotification(msClient.getHiveClient(), eventRequest,
catalog_.getDefaultSkippedHmsEventTypes());
@@ -1435,4 +1467,48 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
public static MessageDeserializer getMessageDeserializer() {
return MESSAGE_DESERIALIZER;
}
+
+ public static class MetaDataFilter {
+ public NotificationFilter filter_;
+ public String catName_;
+ public String dbName_;
+ public String tableName_;
+
+ public MetaDataFilter(NotificationFilter notificationFilter) {
+ this.filter_ = notificationFilter; // if this null then don't build event filter
+ }
+
+ public MetaDataFilter(NotificationFilter notificationFilter, String catName,
+ String dbName) {
+ this(notificationFilter);
+ this.catName_ = Preconditions.checkNotNull(catName);
+ this.dbName_ = Preconditions.checkNotNull(dbName);
+ }
+
+ public MetaDataFilter(NotificationFilter notificationFilter, String catName,
+ String databaseName, String tblName) {
+ this(notificationFilter, catName, databaseName);
+ this.tableName_ = tblName;
+ }
+
+ public void setNotificationFilter(NotificationFilter notificationFilter) {
+ this.filter_ = notificationFilter;
+ }
+
+ public NotificationFilter getNotificationFilter() {
+ return filter_;
+ }
+
+ public String getCatName() {
+ return catName_;
+ }
+
+ public String getDbName() {
+ return dbName_;
+ }
+
+ public String getTableName() {
+ return tableName_;
+ }
+ }
}
diff --git a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index 69b8a084d..157663c5f 100644
--- a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.hadoop.io.IOUtils;
import org.apache.impala.analysis.FunctionName;
@@ -96,6 +97,7 @@ import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventFactory;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventPropertyKey;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType;
import org.apache.impala.catalog.events.MetastoreEventsProcessor.EventProcessorStatus;
+import org.apache.impala.catalog.events.MetastoreEventsProcessor.MetaDataFilter;
import org.apache.impala.catalog.FileMetadataLoadOpts;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
@@ -3962,6 +3964,80 @@ public class MetastoreEventsProcessorTest {
}
}
+ @Test
+ public void testNotificationEventRequest() throws Exception {
+ long currentEventId = eventsProcessor_.getCurrentEventId();
+ int EVENTS_BATCH_SIZE_PER_RPC = 1000;
+ // Generate some DB only related events
+ createDatabaseFromImpala(TEST_DB_NAME, null);
+ String testDbParamKey = "testKey", testDbParamVal = "testVal";
+ String testTable1 = "testNotifyTable1", testTable2 = "testNotifyTable2";
+ addDatabaseParameters(testDbParamKey, testDbParamVal);
+ eventsProcessor_.processEvents();
+ // Verify DB related events only.
+
+ // Generate some table events for managed table
+ testInsertIntoTransactionalTable(testTable1, true, false);
+ alterTableAddColsFromImpala(
+ TEST_DB_NAME, testTable1, "newCol", TPrimitiveType.STRING);
+ alterTableRemoveColFromImpala(TEST_DB_NAME, testTable1, "newCol");
+ eventsProcessor_.processEvents();
+
+ // Generate some table events for external table
+ createTable(testTable2, false);
+ eventsProcessor_.processEvents();
+ alterTableSetOwnerFromImpala(TEST_DB_NAME, testTable2, "testowner");
+ alterTableSetFileFormatFromImpala(
+ TEST_DB_NAME, testTable2, THdfsFileFormat.TEXT);
+ eventsProcessor_.processEvents();
+
+ // verify DB only events
+ NotificationFilter filter = event ->
+ MetastoreEvents.CreateDatabaseEvent.EVENT_TYPE
+ .equals(event.getEventType());
+ MetaDataFilter metaDataFilter = new MetaDataFilter(filter,
+ MetastoreShim.getDefaultCatalogName(), TEST_DB_NAME);
+ assertEquals(MetastoreEventsProcessor
+ .getNextMetastoreEventsWithFilterInBatches(catalog_, currentEventId,
+ metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 1);
+ Db db = catalog_.getDb(TEST_DB_NAME);
+ metaDataFilter.setNotificationFilter(
+ MetastoreEventsProcessor.getDbNotificationEventFilter(db));
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 2);
+
+ // verify events for table 1
+ filter = event -> "ALTER_TABLE".equals(event.getEventType());
+ metaDataFilter = new MetaDataFilter(filter, MetastoreShim.getDefaultCatalogName(),
+ TEST_DB_NAME, testTable1);
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 2);
+ metaDataFilter.setNotificationFilter(null);
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 3);
+
+ // verify events for table 2
+ filter = event -> "ALTER_TABLE".equals(event.getEventType());
+ metaDataFilter = new MetaDataFilter(filter, MetastoreShim.getDefaultCatalogName(),
+ TEST_DB_NAME, testTable2);
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 2);
+ metaDataFilter.setNotificationFilter(null);
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 3);
+
+ // verify all events
+ filter = event -> "ALTER_TABLE".equals(event.getEventType());
+ metaDataFilter = new MetaDataFilter(filter, MetastoreShim.getDefaultCatalogName(),
+ TEST_DB_NAME);
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 4);
+ metaDataFilter.setNotificationFilter(null);
+ // 2 DB + 6 table events
+ assertEquals(MetastoreEventsProcessor.getNextMetastoreEventsWithFilterInBatches(
+ catalog_, currentEventId, metaDataFilter, EVENTS_BATCH_SIZE_PER_RPC).size(), 8);
+ }
+
private void createDatabase(String catName, String dbName,
Map<String, String> params) throws TException {
try(MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
diff --git a/fe/src/test/java/org/apache/impala/catalog/metastore/CatalogHmsSyncToLatestEventIdTest.java b/fe/src/test/java/org/apache/impala/catalog/metastore/CatalogHmsSyncToLatestEventIdTest.java
index 762c6fe69..fa0c71e17 100644
--- a/fe/src/test/java/org/apache/impala/catalog/metastore/CatalogHmsSyncToLatestEventIdTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/metastore/CatalogHmsSyncToLatestEventIdTest.java
@@ -459,7 +459,7 @@ public class CatalogHmsSyncToLatestEventIdTest extends AbstractCatalogMetastoreT
@Test
public void testAlterTableRename() throws Exception {
LOG.info("Executing testALterTableRename");
- String tblName = "test_alter_table_rename_" + tableType_ + "_tbl";
+ String tblName = ("test_alter_table_rename_" + tableType_ + "_tbl").toLowerCase();
String newTblName = tblName + "_new";
try {
createDatabaseInCatalog(TEST_DB_NAME);
diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
index f0bce94ac..d6c7535a8 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -92,6 +92,8 @@ the same dependencies
<include>org/apache/hive/common/util/TxnIdUtils*</include>
<!-- Needed to support describe formatted command compat with Hive -->
<include>org/apache/hadoop/hive/ql/metadata/**/*</include>
+ <!-- Needed to support describe formatted command since HIVE-24509 -->
+ <include>org/apache/hadoop/hive/ql/ddl/ShowUtils.class</include>
<include>org/apache/hadoop/hive/ql/parse/SemanticException.class</include>
<!-- Needed to support Hive udfs -->
<include>org/apache/hadoop/hive/ql/exec/*UDF*</include>
(impala) 02/02: IMPALA-13018: Block push down of conjuncts with implicit casting on base columns for jdbc tables
Posted by wz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3cbb3be5f72dbb889744675fa109dbd1659a7a84
Author: wzhou-code <wz...@cloudera.com>
AuthorDate: Tue May 7 00:19:28 2024 -0700
IMPALA-13018: Block push down of conjuncts with implicit casting on base columns for jdbc tables
The query of q80a consists BETWEEN with casting to timestamp in where
clause like:
d_date between cast('2000-08-23' as timestamp)
and (cast('2000-08-23' as timestamp) + interval 30 days)
Between predicate does cast all exprs to compatible types. Planner
generates predicates for DataSourceScanNode as:
CAST(d_date AS TIMESTAMP) >= TIMESTAMP '2000-08-23 00:00:00',
CAST(d_date AS TIMESTAMP) <= TIMESTAMP '2000-09-22 00:00:00'
But casting to Date/Timestamp for a column cannot be pushed down to JDBC
table now. This patch fixes the issue by blocking such conjuncts with
implicit unsafe casting or casting to date/timestamp to be added into
offered predicate list for JDBC table.
Note that explicit casting on base columns are not allowed to
pushdown.
Testing:
- Add new planner unit-tests, including explicit casting, implicit
casting to date/timestamp, built-in functions, arithmetic
expressions.
The predicates which are accepted for JDBC are shown in plan under
"data source predicates" of DataSourceScanNode, predicates which
are not accepted for JDBC are shown in plan under "predicates" of
DataSourceScanNodes.
- Passed all tpcds queries for JDBC tables, including q80a.
- Passed core test
Change-Id: Iabd7e28b8d5f11f25a000dc4c9ab65895056b572
Reviewed-on: http://gerrit.cloudera.org:8080/21409
Reviewed-by: Riza Suminto <ri...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/planner/DataSourceScanNode.java | 32 ++-
.../queries/PlannerTest/constant-folding.test | 4 +-
.../queries/PlannerTest/data-source-tables.test | 269 ++++++++++++++++++++-
.../queries/PlannerTest/small-query-opt.test | 2 +-
tests/query_test/test_tpcds_queries.py | 5 +-
5 files changed, 287 insertions(+), 25 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
index 9d58861d6..e65225b13 100644
--- a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.BinaryPredicate;
import org.apache.impala.analysis.BoolLiteral;
+import org.apache.impala.analysis.CastExpr;
import org.apache.impala.analysis.CompoundPredicate;
import org.apache.impala.analysis.DateLiteral;
import org.apache.impala.analysis.Expr;
@@ -273,23 +274,42 @@ public class DataSourceScanNode extends ScanNode {
List<TBinaryPredicate> predicates) {
if (conjunct instanceof BinaryPredicate) {
if (conjunct.getChildren().size() != 2) return false;
+ Expr colExpr = null;
SlotRef slotRef = null;
LiteralExpr literalExpr = null;
TComparisonOp op = null;
if ((conjunct.getChild(0).unwrapSlotRef(true) instanceof SlotRef) &&
(conjunct.getChild(1) instanceof LiteralExpr)) {
- slotRef = conjunct.getChild(0).unwrapSlotRef(true);
+ colExpr = conjunct.getChild(0);
+ slotRef = colExpr.unwrapSlotRef(true);
literalExpr = (LiteralExpr) conjunct.getChild(1);
op = ((BinaryPredicate) conjunct).getOp().getThriftOp();
} else if ((conjunct.getChild(1).unwrapSlotRef(true) instanceof SlotRef) &&
(conjunct.getChild(0) instanceof LiteralExpr)) {
- slotRef = conjunct.getChild(1).unwrapSlotRef(true);
+ colExpr = conjunct.getChild(1);
+ slotRef = colExpr.unwrapSlotRef(true);
literalExpr = (LiteralExpr) conjunct.getChild(0);
op = ((BinaryPredicate) conjunct).getOp().converse().getThriftOp();
} else {
+ // Other binary predicate scenarios are not accepted, including:
+ // both operands are slot references, one or both of the operands is an
+ // expression using SlotRef such as explicit CAST, built-in functions,
+ // arithmetic expressions, etc.
return false;
}
+ if (colExpr instanceof CastExpr) {
+ CastExpr castExpr = (CastExpr)colExpr;
+ Preconditions.checkNotNull(castExpr.getType());
+ if (castExpr.getType().isDateOrTimeType()
+ || castExpr.getCompatibility().isUnsafe()) {
+ // Unsafe casting or casting to Date/Timestamp for a column cannot be pushed
+ // down to JDBC table since TBinaryPredicate does not have column Expr.
+ // Return false here to avoid the conjunct to be added to offered predicate
+ // list.
+ return false;
+ }
+ }
TColumnValue val = literalToColumnValue(literalExpr);
if (val == null) return false; // false if unsupported type, e.g.
@@ -408,12 +428,12 @@ public class DataSourceScanNode extends ScanNode {
output.append(String.format("%s%s:%s [%s%s]\n", prefix, id_.toString(),
displayName_, table_.getFullName(), aliasStr));
- if (!acceptedConjuncts_.isEmpty()) {
- output.append(prefix + "data source predicates: "
+ if (acceptedConjuncts_ != null && !acceptedConjuncts_.isEmpty()) {
+ output.append(detailPrefix + "data source predicates: "
+ Expr.getExplainString(acceptedConjuncts_, detailLevel) + "\n");
}
- if (!conjuncts_.isEmpty()) {
- output.append(prefix + "predicates: " +
+ if (conjuncts_ != null && !conjuncts_.isEmpty()) {
+ output.append(detailPrefix + "predicates: " +
Expr.getExplainString(conjuncts_, detailLevel) + "\n");
}
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
index 2baf1d42a..90a70cc27 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
@@ -116,8 +116,8 @@ PLAN-ROOT SINK
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
|
00:SCAN DATA SOURCE [functional.alltypes_datasource]
-data source predicates: CAST(tinyint_col AS DOUBLE) < CAST(256 AS DOUBLE), int_col < CAST(2 AS INT)
-predicates: float_col != CAST(0 AS FLOAT)
+ data source predicates: CAST(tinyint_col AS DOUBLE) < CAST(256 AS DOUBLE), int_col < CAST(2 AS INT)
+ predicates: float_col != CAST(0 AS FLOAT)
mem-estimate=1.00GB mem-reservation=0B thread-reservation=0
tuple-ids=0 row-size=116B cardinality=500
in pipelines: 00(GETNEXT)
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test b/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test
index 6256c8e8f..5d43ff760 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test
@@ -8,8 +8,8 @@ where tinyint_col < 256 and
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_datasource]
-data source predicates: tinyint_col < 256
-predicates: float_col != 0, CAST(int_col AS BIGINT) < 10
+ data source predicates: tinyint_col < 256
+ predicates: float_col != 0, CAST(int_col AS BIGINT) < 10
row-size=116B cardinality=500
====
# The first four predicates are in a form that can be offered to the data source
@@ -28,8 +28,8 @@ where 10 > int_col and
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_datasource]
-data source predicates: int_col < 10, string_col != 'Foo'
-predicates: double_col < 5, NOT bool_col = TRUE, NOT double_col = 5.0, string_col != 'Bar'
+ data source predicates: int_col < 10, string_col != 'Foo'
+ predicates: double_col < 5, NOT bool_col = TRUE, NOT double_col = 5.0, string_col != 'Bar'
row-size=116B cardinality=500
====
# The 3rd predicate is not in a form that can be offered to the data source so
@@ -43,8 +43,8 @@ where int_col < 10 and
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_datasource]
-data source predicates: int_col < 10, bool_col != FALSE
-predicates: double_col > 5, string_col IN ('Foo', 'Bar')
+ data source predicates: int_col < 10, bool_col != FALSE
+ predicates: double_col > 5, string_col IN ('Foo', 'Bar')
row-size=116B cardinality=500
====
# Tests that all predicates from the On-clause are applied (IMPALA-805)
@@ -66,11 +66,11 @@ PLAN-ROOT SINK
| row-size=35B cardinality=500
|
|--01:SCAN DATA SOURCE [functional.alltypes_datasource b]
-|--predicates: b.id = b.int_col, b.id = b.bigint_col
+| predicates: b.id = b.int_col, b.id = b.bigint_col
| row-size=0B cardinality=500
|
00:SCAN DATA SOURCE [functional.alltypes_datasource a]
-predicates: a.id = a.int_col, a.id = a.tinyint_col, a.int_col = a.bigint_col, a.tinyint_col = a.smallint_col
+ predicates: a.id = a.int_col, a.id = a.tinyint_col, a.int_col = a.bigint_col, a.tinyint_col = a.smallint_col
row-size=0B cardinality=500
====
# Tests that <=>, IS DISTINCT FROM, and IS NOT DISTINCT FROM all can be offered to the
@@ -86,8 +86,8 @@ and bigint_col is not distinct from 5
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_datasource]
-data source predicates: id IS NOT DISTINCT FROM 1, tinyint_col IS DISTINCT FROM 2, int_col IS NOT DISTINCT FROM 4
-predicates: bigint_col IS NOT DISTINCT FROM 5, bool_col IS NOT DISTINCT FROM TRUE, smallint_col IS DISTINCT FROM 3
+ data source predicates: id IS NOT DISTINCT FROM 1, tinyint_col IS DISTINCT FROM 2, int_col IS NOT DISTINCT FROM 4
+ predicates: bigint_col IS NOT DISTINCT FROM 5, bool_col IS NOT DISTINCT FROM TRUE, smallint_col IS DISTINCT FROM 3
row-size=116B cardinality=500
====
# EmptySet datasource
@@ -104,32 +104,275 @@ PLAN-ROOT SINK
00:EMPTYSET
====
# IMPALA-12815: Support timestamp as external datasource scan predicates
+# Date type of predicate can be offered to jdbc data source.
select * from functional.alltypes_jdbc_datasource where date_col > DATE '2009-01-02'
and timestamp_col > '2009-01-02 00:12:00' limit 15
---- PLAN
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_jdbc_datasource]
-data source predicates: date_col > DATE '2009-01-02', timestamp_col > TIMESTAMP '2009-01-02 00:12:00'
+ data source predicates: date_col > DATE '2009-01-02', timestamp_col > TIMESTAMP '2009-01-02 00:12:00'
limit: 15
row-size=64B cardinality=1
====
# IMPALA-12815: use timestamp with cast() and to_timestamp()
+# Timestamp type of predicate can be offered to jdbc data source.
select * from functional.alltypes_jdbc_datasource where timestamp_col between
date_add(cast('2009-01-01' as timestamp), interval 1 day) and to_timestamp('Jan 04, 2009', 'MMM dd, yyyy') limit 15;
---- PLAN
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_jdbc_datasource]
-data source predicates: timestamp_col >= TIMESTAMP '2009-01-02 00:00:00', timestamp_col <= TIMESTAMP '2009-01-04 00:00:00'
+ data source predicates: timestamp_col >= TIMESTAMP '2009-01-02 00:00:00', timestamp_col <= TIMESTAMP '2009-01-04 00:00:00'
limit: 15
row-size=64B cardinality=1
====
+# Decimal type of predicate can be offered to jdbc data source.
select * from functional.jdbc_decimal_tbl where d3 > 123.456 and d5 < 10.0
---- PLAN
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.jdbc_decimal_tbl]
-data source predicates: d3 > 123.456, d5 < 10.0
+ data source predicates: d3 > 123.456, d5 < 10.0
row-size=52B cardinality=1
====
+# Implicit casting date_dim.d_date to date cannot be offered to jdbc data source.
+select s_store_id as store_id,
+ sum(ss_ext_sales_price) as sales,
+ sum(coalesce(sr_return_amt, 0)) as return_amt,
+ sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit
+ from tpcds_jdbc.store_sales left outer join tpcds_jdbc.store_returns on
+ (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number),
+ tpcds_jdbc.date_dim,
+ tpcds_jdbc.store,
+ tpcds_jdbc.item,
+ tpcds_jdbc.promotion
+ where ss_sold_date_sk = d_date_sk
+ and d_date between cast('2000-08-23' as date)
+ and (cast('2000-08-23' as date) + interval 30 days)
+ and ss_store_sk = s_store_sk
+ and ss_item_sk = i_item_sk
+ and i_current_price > 50
+ and ss_promo_sk = p_promo_sk
+ and p_channel_tv = 'N'
+ group by s_store_id;
+---- PLAN
+PLAN-ROOT SINK
+|
+11:AGGREGATE [FINALIZE]
+| output: sum(ss_ext_sales_price), sum(coalesce(sr_return_amt, 0)), sum(ss_net_profit - coalesce(sr_net_loss, 0))
+| group by: s_store_id
+| row-size=60B cardinality=1
+|
+10:HASH JOIN [INNER JOIN]
+| hash predicates: ss_promo_sk = p_promo_sk
+| row-size=104B cardinality=1
+|
+|--05:SCAN DATA SOURCE [tpcds_jdbc.promotion]
+| data source predicates: p_channel_tv = 'N'
+| row-size=0B cardinality=1
+|
+09:HASH JOIN [INNER JOIN]
+| hash predicates: ss_item_sk = i_item_sk
+| row-size=100B cardinality=1
+|
+|--04:SCAN DATA SOURCE [tpcds_jdbc.item]
+| data source predicates: i_current_price > 50
+| row-size=0B cardinality=1
+|
+08:HASH JOIN [INNER JOIN]
+| hash predicates: ss_store_sk = s_store_sk
+| row-size=92B cardinality=1
+|
+|--03:SCAN DATA SOURCE [tpcds_jdbc.store]
+| row-size=0B cardinality=1
+|
+07:HASH JOIN [INNER JOIN]
+| hash predicates: ss_sold_date_sk = d_date_sk
+| row-size=76B cardinality=1
+|
+|--02:SCAN DATA SOURCE [tpcds_jdbc.date_dim]
+| predicates: d_date <= DATE '2000-09-22', d_date >= DATE '2000-08-23'
+| row-size=0B cardinality=1
+|
+06:HASH JOIN [LEFT OUTER JOIN]
+| hash predicates: ss_item_sk = sr_item_sk, ss_ticket_number = sr_ticket_number
+| row-size=60B cardinality=1
+|
+|--01:SCAN DATA SOURCE [tpcds_jdbc.store_returns]
+| row-size=0B cardinality=1
+|
+00:SCAN DATA SOURCE [tpcds_jdbc.store_sales]
+ row-size=0B cardinality=1
+====
+# Explicit casting date_dim.d_date to timestamp cannot be offered to jdbc data source.
+select s_store_id as store_id,
+ sum(ss_ext_sales_price) as sales,
+ sum(coalesce(sr_return_amt, 0)) as return_amt,
+ sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit
+ from tpcds_jdbc.store_sales left outer join tpcds_jdbc.store_returns on
+ (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number),
+ tpcds_jdbc.date_dim,
+ tpcds_jdbc.store,
+ tpcds_jdbc.item,
+ tpcds_jdbc.promotion
+ where ss_sold_date_sk = d_date_sk
+ and cast(d_date as timestamp) between cast('2000-08-23' as timestamp)
+ and (cast('2000-08-23' as timestamp) + interval 30 days)
+ and ss_store_sk = s_store_sk
+ and ss_item_sk = i_item_sk
+ and i_current_price > 50
+ and ss_promo_sk = p_promo_sk
+ and p_channel_tv = 'N'
+ group by s_store_id;
+---- PLAN
+PLAN-ROOT SINK
+|
+11:AGGREGATE [FINALIZE]
+| output: sum(ss_ext_sales_price), sum(coalesce(sr_return_amt, 0)), sum(ss_net_profit - coalesce(sr_net_loss, 0))
+| group by: s_store_id
+| row-size=60B cardinality=1
+|
+10:HASH JOIN [INNER JOIN]
+| hash predicates: ss_promo_sk = p_promo_sk
+| row-size=104B cardinality=1
+|
+|--05:SCAN DATA SOURCE [tpcds_jdbc.promotion]
+| data source predicates: p_channel_tv = 'N'
+| row-size=0B cardinality=1
+|
+09:HASH JOIN [INNER JOIN]
+| hash predicates: ss_item_sk = i_item_sk
+| row-size=100B cardinality=1
+|
+|--04:SCAN DATA SOURCE [tpcds_jdbc.item]
+| data source predicates: i_current_price > 50
+| row-size=0B cardinality=1
+|
+08:HASH JOIN [INNER JOIN]
+| hash predicates: ss_store_sk = s_store_sk
+| row-size=92B cardinality=1
+|
+|--03:SCAN DATA SOURCE [tpcds_jdbc.store]
+| row-size=0B cardinality=1
+|
+07:HASH JOIN [INNER JOIN]
+| hash predicates: ss_sold_date_sk = d_date_sk
+| row-size=76B cardinality=1
+|
+|--02:SCAN DATA SOURCE [tpcds_jdbc.date_dim]
+| predicates: CAST(d_date AS TIMESTAMP) <= TIMESTAMP '2000-09-22 00:00:00', CAST(d_date AS TIMESTAMP) >= TIMESTAMP '2000-08-23 00:00:00'
+| row-size=0B cardinality=1
+|
+06:HASH JOIN [LEFT OUTER JOIN]
+| hash predicates: ss_item_sk = sr_item_sk, ss_ticket_number = sr_ticket_number
+| row-size=60B cardinality=1
+|
+|--01:SCAN DATA SOURCE [tpcds_jdbc.store_returns]
+| row-size=0B cardinality=1
+|
+00:SCAN DATA SOURCE [tpcds_jdbc.store_sales]
+ row-size=0B cardinality=1
+====
+# Implicit casting date_dim.d_date to timestamp cannot be offered to jdbc data source.
+select s_store_id as store_id,
+ sum(ss_ext_sales_price) as sales,
+ sum(coalesce(sr_return_amt, 0)) as return_amt,
+ sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit
+ from tpcds_jdbc.store_sales left outer join tpcds_jdbc.store_returns on
+ (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number),
+ tpcds_jdbc.date_dim,
+ tpcds_jdbc.store,
+ tpcds_jdbc.item,
+ tpcds_jdbc.promotion
+ where ss_sold_date_sk = d_date_sk
+ and d_date between cast('2000-08-23' as timestamp)
+ and (cast('2000-08-23' as timestamp) + interval 30 days)
+ and ss_store_sk = s_store_sk
+ and ss_item_sk = i_item_sk
+ and i_current_price > 50
+ and ss_promo_sk = p_promo_sk
+ and p_channel_tv = 'N'
+ group by s_store_id;
+---- PLAN
+PLAN-ROOT SINK
+|
+11:AGGREGATE [FINALIZE]
+| output: sum(ss_ext_sales_price), sum(coalesce(sr_return_amt, 0)), sum(ss_net_profit - coalesce(sr_net_loss, 0))
+| group by: s_store_id
+| row-size=60B cardinality=1
+|
+10:HASH JOIN [INNER JOIN]
+| hash predicates: ss_promo_sk = p_promo_sk
+| row-size=104B cardinality=1
+|
+|--05:SCAN DATA SOURCE [tpcds_jdbc.promotion]
+| data source predicates: p_channel_tv = 'N'
+| row-size=0B cardinality=1
+|
+09:HASH JOIN [INNER JOIN]
+| hash predicates: ss_item_sk = i_item_sk
+| row-size=100B cardinality=1
+|
+|--04:SCAN DATA SOURCE [tpcds_jdbc.item]
+| data source predicates: i_current_price > 50
+| row-size=0B cardinality=1
+|
+08:HASH JOIN [INNER JOIN]
+| hash predicates: ss_store_sk = s_store_sk
+| row-size=92B cardinality=1
+|
+|--03:SCAN DATA SOURCE [tpcds_jdbc.store]
+| row-size=0B cardinality=1
+|
+07:HASH JOIN [INNER JOIN]
+| hash predicates: ss_sold_date_sk = d_date_sk
+| row-size=76B cardinality=1
+|
+|--02:SCAN DATA SOURCE [tpcds_jdbc.date_dim]
+| predicates: d_date <= TIMESTAMP '2000-09-22 00:00:00', d_date >= TIMESTAMP '2000-08-23 00:00:00'
+| row-size=0B cardinality=1
+|
+06:HASH JOIN [LEFT OUTER JOIN]
+| hash predicates: ss_item_sk = sr_item_sk, ss_ticket_number = sr_ticket_number
+| row-size=60B cardinality=1
+|
+|--01:SCAN DATA SOURCE [tpcds_jdbc.store_returns]
+| row-size=0B cardinality=1
+|
+00:SCAN DATA SOURCE [tpcds_jdbc.store_sales]
+ row-size=0B cardinality=1
+====
+# Arithmetic expression cannot be offered to jdbc data source.
+select d_year from tpcds_jdbc.date_dim where (d_date_sk + 2) < 2515030;
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN DATA SOURCE [tpcds_jdbc.date_dim]
+ predicates: (d_date_sk + 2) < 2515030
+ row-size=8B cardinality=1
+====
+# Built-in function cannot be offered to jdbc data source.
+select d_year from tpcds_jdbc.date_dim where length(d_date_id) <= 16;
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN DATA SOURCE [tpcds_jdbc.date_dim]
+ predicates: length(d_date_id) <= 16
+ row-size=16B cardinality=1
+====
+# Explicit casting int_col cannot be offered to jdbc data source,
+# implicit casting float_col and double_col can be offered to jdbc data source.
+select * from functional.alltypes_jdbc_datasource
+where tinyint_col < 256 and
+ float_col != cast(0 as double) and
+ double_col > cast(5 as float) and
+ cast(int_col as bigint) < 10
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN DATA SOURCE [functional.alltypes_jdbc_datasource]
+ data source predicates: tinyint_col < 256, float_col != 0, double_col > 5
+ predicates: CAST(int_col AS BIGINT) < 10
+ row-size=64B cardinality=1
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/small-query-opt.test b/testdata/workloads/functional-planner/queries/PlannerTest/small-query-opt.test
index 358f4f960..42c826f52 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/small-query-opt.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/small-query-opt.test
@@ -636,7 +636,7 @@ select * from functional.alltypes_datasource where id = 1 limit 7
PLAN-ROOT SINK
|
00:SCAN DATA SOURCE [functional.alltypes_datasource]
-data source predicates: id = 1
+ data source predicates: id = 1
limit: 7
row-size=116B cardinality=7
====
diff --git a/tests/query_test/test_tpcds_queries.py b/tests/query_test/test_tpcds_queries.py
index c9fd5a821..7e3e3e3fc 100644
--- a/tests/query_test/test_tpcds_queries.py
+++ b/tests/query_test/test_tpcds_queries.py
@@ -1078,9 +1078,8 @@ class TestTpcdsQueryForJdbcTables(ImpalaTestSuite):
def test_tpcds_q80(self, vector):
self.run_test_case('tpcds-decimal_v2-q80', vector, use_db='tpcds_jdbc')
- # TODO: IMPALA-13018 fix unmatched results for external JDBC tables.
- # def test_tpcds_q80a(self, vector):
- # self.run_test_case('tpcds-decimal_v2-q80a', vector, use_db='tpcds_jdbc')
+ def test_tpcds_q80a(self, vector):
+ self.run_test_case('tpcds-decimal_v2-q80a', vector, use_db='tpcds_jdbc')
def test_tpcds_q81(self, vector):
self.run_test_case('tpcds-decimal_v2-q81', vector, use_db='tpcds_jdbc')