You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/07/19 11:01:13 UTC
[impala] 04/04: IMPALA-10871 (part 2): Apache Hive 3: fixes for dataset loading
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1d16367afb1e520bae5992c1e6c92b4fb41d2655
Author: Fucun Chu <ch...@hotmail.com>
AuthorDate: Mon Nov 15 19:31:30 2021 +0800
IMPALA-10871 (part 2): Apache Hive 3: fixes for dataset loading
This patch fixes the data loading problem of integrating Apache Hive 3
and switches to the tez engine.
Add HIVE-21569, HIVE-20038 patches and recompile the hive-exec module.
Testing:
- Manually perform data loading steps.
Change-Id: I86a1fdffc70b8d9a3bc97a72b5b939021dc496f1
Reviewed-on: http://gerrit.cloudera.org:8080/18028
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/compat/MetastoreShim.java | 19 ++
.../org/apache/impala/compat/MetastoreShim.java | 7 +
.../apache/impala/service/CatalogOpExecutor.java | 2 +
fe/src/test/resources/hive-site.xml.py | 1 +
testdata/bin/generate-schema-statements.py | 4 +
testdata/bin/load_nested.py | 3 +-
testdata/bin/patch_hive.sh | 9 +-
testdata/cluster/hive/README | 6 +
testdata/cluster/hive/patch1-HIVE-21569.diff | 257 +++++++++++++++++++++
testdata/cluster/hive/patch2-HIVE-20038.diff | 67 ++++++
tests/util/test_file_parser.py | 8 +
11 files changed, 381 insertions(+), 2 deletions(-)
diff --git a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index bc5a98fd9..1c06b502e 100644
--- a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogServiceCatalog;
+import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.Hive3MetastoreShimBase;
@@ -476,6 +477,24 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
return null;
}
+ /**
+ * Set the default table path for a new table.
+ */
+ public static void setTableLocation(Db db, Table tbl) throws ImpalaRuntimeException {
+ Preconditions.checkNotNull(db);
+ Preconditions.checkNotNull(tbl);
+ // Set a valid location of this table using the same rules as the cdp metastore,
+ // unless the user specified a path.
+ try {
+ if (tbl.getSd().getLocation() == null || tbl.getSd().getLocation().isEmpty()) {
+ tbl.getSd().setLocation(
+ MetastoreShim.getPathForNewTable(db.getMetaStoreDb(), tbl));
+ }
+ } catch (MetaException e) {
+ throw new ImpalaRuntimeException("setTableLocation", e);
+ }
+ }
+
/**
* Wrapper around HMS Table object to get table id.
*/
diff --git a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index b29f5a34c..8cc30a4c0 100644
--- a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -84,6 +84,7 @@ import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogServiceCatalog;
import org.apache.impala.catalog.CompactionInfoLoader;
import org.apache.impala.catalog.DatabaseNotFoundException;
+import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.Hive3MetastoreShimBase;
@@ -517,6 +518,12 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
return db.getManagedLocationUri();
}
+ /**
+ * Apache Hive-3 only function.
+ */
+ public static void setTableLocation(Db db, Table tbl) throws ImpalaRuntimeException {
+ }
+
/**
* Wrapper around HMS Table object to get table id.
*/
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 0dbb3823e..93a17623e 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -3189,6 +3189,7 @@ public class CatalogOpExecutor {
}
Preconditions.checkState(params.getColumns().size() > 0,
"Empty column list given as argument to Catalog.createTable");
+ MetastoreShim.setTableLocation(catalog_.getDb(tbl.getDbName()), tbl);
return createTable(tbl, params.if_not_exists, params.getCache_op(),
params.server_name, params.getPrimary_keys(), params.getForeign_keys(),
wantMinimalResult, response);
@@ -3772,6 +3773,7 @@ public class CatalogOpExecutor {
createIcebergTable(tbl, wantMinimalResult, response, params.if_not_exists, columns,
partitionSpec, tableProperties, params.getComment());
} else {
+ MetastoreShim.setTableLocation(catalog_.getDb(tbl.getDbName()), tbl);
createTable(tbl, params.if_not_exists, null, params.server_name, null, null,
wantMinimalResult, response);
}
diff --git a/fe/src/test/resources/hive-site.xml.py b/fe/src/test/resources/hive-site.xml.py
index 80c758beb..db1ba97b2 100644
--- a/fe/src/test/resources/hive-site.xml.py
+++ b/fe/src/test/resources/hive-site.xml.py
@@ -112,6 +112,7 @@ if kerberize:
# Enable Tez, ACID and proleptic Gregorian calendar DATE types for Hive 3
if hive_major_version >= 3:
CONFIG.update({
+ 'hive.execution.engine': 'tez',
'hive.tez.container.size': '512',
'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager',
# We run YARN with Tez on the classpath directly
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 10a68e543..00bc9bf6a 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -482,6 +482,10 @@ def build_insert_into_statement(insert, db_name, db_suffix, table_name, file_for
def build_hbase_insert(db_name, db_suffix, table_name):
hbase_insert = SET_HIVE_HBASE_BULK_LOAD + ';\n'
+ # For Apache Hive, "hive.hbase.bulk does not exist" exception will be thrown and there
+ # is only warning in cdp
+ if os.environ['USE_APACHE_HIVE'] == "true":
+ hbase_insert = ""
hbase_insert += ("INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name}"
" SELECT * FROM {db_name}.{table_name};\n").\
format(db_name=db_name, db_suffix=db_suffix,table_name=table_name)
diff --git a/testdata/bin/load_nested.py b/testdata/bin/load_nested.py
index 76a43b1b9..a4ae42646 100755
--- a/testdata/bin/load_nested.py
+++ b/testdata/bin/load_nested.py
@@ -90,7 +90,8 @@ def load():
# explicitly creating an external table so that files are in the external warehouse
# directory. Use external.table.purge=true so that it is equivalent to a Hive 2
# managed table.
- if HIVE_MAJOR_VERSION >= 3:
+ # For Apache Hive, HIVE-20085 (Hive 4) Allow CTAS.
+ if HIVE_MAJOR_VERSION >= 3 and os.environ["USE_APACHE_HIVE"] != "true":
external = "EXTERNAL"
tblproperties += ",'external.table.purge'='TRUE'"
sql_params = {
diff --git a/testdata/bin/patch_hive.sh b/testdata/bin/patch_hive.sh
index fa71e46ca..c87cee11c 100755
--- a/testdata/bin/patch_hive.sh
+++ b/testdata/bin/patch_hive.sh
@@ -32,6 +32,8 @@ if [[ "${USE_APACHE_HIVE}" != true ]]; then
exit 0
fi
+# Rebuild tag
+HIVE_REBUILD=${HIVE_REBUILD-false}
# Cache applied patches
PATCHED_CACHE_FILE="$HIVE_SRC_DIR/.patched"
if [ ! -f "$PATCHED_CACHE_FILE" ]; then touch "$PATCHED_CACHE_FILE"; fi
@@ -53,6 +55,7 @@ function apply_patch {
echo "Apply patch: $p"
patch -p1 < ${HIVE_PARCH_DIR}/$p
echo $p >> $PATCHED_CACHE_FILE
+ HIVE_REBUILD=true
fi
}
@@ -70,5 +73,9 @@ do
done
# 3. Repackage the hive submodules affected by the patch
-
+if [[ "${HIVE_REBUILD}" = "true" ]]; then
+ echo "Repackage the hive-exec module"
+ ${IMPALA_HOME}/bin/mvn-quiet.sh -pl ql clean package -Dmaven.test.skip
+ cp $HIVE_SRC_DIR/ql/target/hive-exec-${APACHE_HIVE_VERSION}.jar $HIVE_HOME/lib/
+fi
popd
diff --git a/testdata/cluster/hive/README b/testdata/cluster/hive/README
index 9582bf8c7..c80b84227 100644
--- a/testdata/cluster/hive/README
+++ b/testdata/cluster/hive/README
@@ -4,3 +4,9 @@ testing.
patch0-HIVE-21586.diff:
Contains only patches for `hive_metastore.thrift` is used to solve the problem that the
generated cpp file cannot be compiled.
+
+patch1-HIVE-21569.diff:
+Bump guava version to 28.1-jre. Fix HIVE-22717.
+
+patch2-HIVE-20038.diff:
+Update queries on non-bucketed + partitioned tables throws NPE
diff --git a/testdata/cluster/hive/patch1-HIVE-21569.diff b/testdata/cluster/hive/patch1-HIVE-21569.diff
new file mode 100644
index 000000000..b893be927
--- /dev/null
+++ b/testdata/cluster/hive/patch1-HIVE-21569.diff
@@ -0,0 +1,257 @@
+diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
+index 27f8a9bd9c..bf39f6acfc 100644
+--- a/druid-handler/pom.xml
++++ b/druid-handler/pom.xml
+@@ -30,7 +30,7 @@
+ <properties>
+ <hive.path.to.root>..</hive.path.to.root>
+ <druid.metamx.util.version>1.3.2</druid.metamx.util.version>
+- <druid.guava.version>16.0.1</druid.guava.version>
++ <druid.guava.version>28.1-jre</druid.guava.version>
+ </properties>
+
+ <dependencies>
+diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java
+index 64c640f45a..6b9f08cd8a 100644
+--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java
++++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java
+@@ -28,6 +28,7 @@
+ import com.google.common.collect.Iterators;
+
+ import java.io.IOException;
++import java.util.Collections;
+ import java.util.Iterator;
+ import java.util.List;
+
+@@ -43,7 +44,7 @@
+
+ private ScanResultValue current;
+
+- private Iterator<List<Object>> compactedValues = Iterators.emptyIterator();
++ private Iterator<List<Object>> compactedValues = Collections.emptyIterator();
+
+ @Override
+ protected JavaType getResultTypeDef() {
+diff --git a/itests/qtest-druid/pom.xml b/itests/qtest-druid/pom.xml
+index 8b20ac342f..ecefff3e52 100644
+--- a/itests/qtest-druid/pom.xml
++++ b/itests/qtest-druid/pom.xml
+@@ -41,7 +41,7 @@
+ <druid.jersey.version>1.19.3</druid.jersey.version>
+ <druid.jetty.version>9.3.19.v20170502</druid.jetty.version>
+ <druid.derby.version>10.11.1.1</druid.derby.version>
+- <druid.guava.version>16.0.1</druid.guava.version>
++ <druid.guava.version>28.1-jre</druid.guava.version>
+ <druid.guice.version>4.1.0</druid.guice.version>
+ <kafka.version>0.10.2.0</kafka.version>
+ </properties>
+diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/AsyncPbRpcProxy.java b/llap-common/src/java/org/apache/hadoop/hive/llap/AsyncPbRpcProxy.java
+index ad39963614..475a159e3f 100644
+--- a/llap-common/src/java/org/apache/hadoop/hive/llap/AsyncPbRpcProxy.java
++++ b/llap-common/src/java/org/apache/hadoop/hive/llap/AsyncPbRpcProxy.java
+@@ -171,7 +171,7 @@ public void shutdown() {
+ CallableRequest<T, U> request, LlapNodeId nodeId) {
+ ListenableFuture<U> future = executor.submit(request);
+ Futures.addCallback(future, new ResponseCallback<U>(
+- request.getCallback(), nodeId, this));
++ request.getCallback(), nodeId, this), MoreExecutors.directExecutor());
+ }
+
+ @VisibleForTesting
+@@ -283,7 +283,7 @@ public void onFailure(Throwable t) {
+ LOG.warn("RequestManager shutdown with error", t);
+ }
+ }
+- });
++ }, MoreExecutors.directExecutor());
+ }
+
+ @Override
+@@ -490,4 +490,4 @@ protected abstract ProtocolType createProtocolImpl(Configuration config, String
+ protected abstract void shutdownProtocolImpl(ProtocolType proxy);
+
+ protected abstract String getTokenUser(Token<TokenType> token);
+-}
+\ No newline at end of file
++}
+diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+index 088a5f33c0..a49920eeac 100644
+--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
++++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+@@ -174,7 +174,7 @@ public void onFailure(Throwable t) {
+ Thread.getDefaultUncaughtExceptionHandler().uncaughtException(Thread.currentThread(), t);
+ }
+ }
+- });
++ }, MoreExecutors.directExecutor());
+ // TODO: why is this needed? we could just save the host and port?
+ nodeId = LlapNodeId.getInstance(localAddress.get().getHostName(), localAddress.get().getPort());
+ LOG.info("AMReporter running with DaemonId: {}, NodeId: {}", daemonId, nodeId);
+@@ -274,7 +274,7 @@ public void onFailure(Throwable t) {
+ LOG.warn("Failed to send taskKilled for {}. The attempt will likely time out.",
+ taskAttemptId);
+ }
+- });
++ }, MoreExecutors.directExecutor());
+ }
+
+ public void queryComplete(QueryIdentifier queryIdentifier) {
+@@ -342,7 +342,7 @@ public void onFailure(Throwable t) {
+ amNodeInfo.amNodeId, currentQueryIdentifier, t);
+ queryFailedHandler.queryFailed(currentQueryIdentifier);
+ }
+- });
++ }, MoreExecutors.directExecutor());
+ }
+ }
+ } catch (InterruptedException e) {
+diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapTaskReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapTaskReporter.java
+index 33ade55ee1..848c42a750 100644
+--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapTaskReporter.java
++++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapTaskReporter.java
+@@ -128,7 +128,7 @@ public synchronized void registerTask(RuntimeTask task,
+ sendCounterInterval, maxEventsToGet, requestCounter, containerIdStr, initialEvent,
+ fragmentRequestId, wmCounters);
+ ListenableFuture<Boolean> future = heartbeatExecutor.submit(currentCallable);
+- Futures.addCallback(future, new HeartbeatCallback(errorReporter));
++ Futures.addCallback(future, new HeartbeatCallback(errorReporter), MoreExecutors.directExecutor());
+ }
+
+ /**
+diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
+index 047a55ccae..fac0f77c6a 100644
+--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
++++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
+@@ -175,7 +175,7 @@ public TaskExecutorService(int numExecutors, int waitQueueSize,
+ executionCompletionExecutorService = MoreExecutors.listeningDecorator(
+ executionCompletionExecutorServiceRaw);
+ ListenableFuture<?> future = waitQueueExecutorService.submit(new WaitQueueWorker());
+- Futures.addCallback(future, new WaitQueueWorkerCallback());
++ Futures.addCallback(future, new WaitQueueWorkerCallback(), MoreExecutors.directExecutor());
+ }
+
+ private LlapQueueComparatorBase createComparator(
+diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+index 82179645da..b20e8cda65 100644
+--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
++++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+@@ -744,15 +744,17 @@ public Void call() throws Exception {
+ }, 10000L, TimeUnit.MILLISECONDS);
+
+ nodeEnablerFuture = nodeEnabledExecutor.submit(nodeEnablerCallable);
+- Futures.addCallback(nodeEnablerFuture, new LoggingFutureCallback("NodeEnablerThread", LOG));
++ Futures.addCallback(nodeEnablerFuture, new LoggingFutureCallback("NodeEnablerThread", LOG),
++ MoreExecutors.directExecutor());
+
+ delayedTaskSchedulerFuture =
+ delayedTaskSchedulerExecutor.submit(delayedTaskSchedulerCallable);
+ Futures.addCallback(delayedTaskSchedulerFuture,
+- new LoggingFutureCallback("DelayedTaskSchedulerThread", LOG));
++ new LoggingFutureCallback("DelayedTaskSchedulerThread", LOG), MoreExecutors.directExecutor());
+
+ schedulerFuture = schedulerExecutor.submit(schedulerCallable);
+- Futures.addCallback(schedulerFuture, new LoggingFutureCallback("SchedulerThread", LOG));
++ Futures.addCallback(schedulerFuture, new LoggingFutureCallback("SchedulerThread", LOG),
++ MoreExecutors.directExecutor());
+
+ registry.start();
+ registry.registerStateChangeListener(new NodeStateChangeListener());
+diff --git a/pom.xml b/pom.xml
+index c06f7f81c1..17dd2cf886 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -144,7 +144,7 @@
+ <dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.2</dropwizard-metrics-hadoop-metrics2-reporter.version>
+ <druid.version>0.12.0</druid.version>
+ <flatbuffers.version>1.2.0-3f79e055</flatbuffers.version>
+- <guava.version>19.0</guava.version>
++ <guava.version>28.1-jre</guava.version>
+ <groovy.version>2.4.11</groovy.version>
+ <h2database.version>1.3.166</h2database.version>
+ <hadoop.version>3.1.0</hadoop.version>
+diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
+index 97ba036335..649ef99d9b 100644
+--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
++++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
+@@ -27,6 +27,7 @@
+ import com.google.common.util.concurrent.FutureCallback;
+ import com.google.common.util.concurrent.Futures;
+ import com.google.common.util.concurrent.ListenableFuture;
++import com.google.common.util.concurrent.MoreExecutors;
+ import com.google.common.util.concurrent.SettableFuture;
+ import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+@@ -1092,7 +1093,7 @@ private static int transferSessionsToDestroy(Collection<WmTezSession> source,
+ }
+
+ private void failOnFutureFailure(ListenableFuture<?> future) {
+- Futures.addCallback(future, FATAL_ERROR_CALLBACK);
++ Futures.addCallback(future, FATAL_ERROR_CALLBACK, MoreExecutors.directExecutor());
+ }
+
+ private void queueGetRequestOnMasterThread(
+@@ -1925,7 +1926,7 @@ public SessionInitContext(SettableFuture<WmTezSession> future,
+
+ public void start() throws Exception {
+ ListenableFuture<WmTezSession> getFuture = tezAmPool.getSessionAsync();
+- Futures.addCallback(getFuture, this);
++ Futures.addCallback(getFuture, this, MoreExecutors.directExecutor());
+ }
+
+ @Override
+@@ -1979,7 +1980,7 @@ public void onSuccess(WmTezSession session) {
+ case GETTING: {
+ ListenableFuture<WmTezSession> waitFuture = session.waitForAmRegistryAsync(
+ amRegistryTimeoutMs, timeoutPool);
+- Futures.addCallback(waitFuture, this);
++ Futures.addCallback(waitFuture, this, MoreExecutors.directExecutor());
+ break;
+ }
+ case WAITING_FOR_REGISTRY: {
+diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
+index f5ab981f26..d4296b9b43 100644
+--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
++++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
+@@ -22,6 +22,7 @@
+ import com.google.common.util.concurrent.Futures;
+ import com.google.common.util.concurrent.FutureCallback;
+ import com.google.common.util.concurrent.ListenableFuture;
++import com.google.common.util.concurrent.MoreExecutors;
+ import com.google.common.util.concurrent.SettableFuture;
+ import java.io.IOException;
+ import java.util.concurrent.ScheduledExecutorService;
+@@ -128,7 +129,7 @@ public void onSuccess(Boolean result) {
+ public void onFailure(Throwable t) {
+ future.setException(t);
+ }
+- });
++ }, MoreExecutors.directExecutor());
+ return future;
+ }
+
+diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
+index 884dbfdf31..bd2e51edeb 100644
+--- a/standalone-metastore/pom.xml
++++ b/standalone-metastore/pom.xml
+@@ -70,7 +70,7 @@
+ <derby.version>10.10.2.0</derby.version>
+ <dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.2</dropwizard-metrics-hadoop-metrics2-reporter.version>
+ <dropwizard.version>3.1.0</dropwizard.version>
+- <guava.version>19.0</guava.version>
++ <guava.version>28.1-jre</guava.version>
+ <hadoop.version>3.1.0</hadoop.version>
+ <hikaricp.version>2.6.1</hikaricp.version>
+ <jackson.version>2.9.5</jackson.version>
+diff --git a/storage-api/pom.xml b/storage-api/pom.xml
+index a40feff575..799e541332 100644
+--- a/storage-api/pom.xml
++++ b/storage-api/pom.xml
+@@ -32,7 +32,7 @@
+ <properties>
+ <commons-lang.version>2.6</commons-lang.version>
+ <commons-logging.version>1.1.3</commons-logging.version>
+- <guava.version>19.0</guava.version>
++ <guava.version>28.1-jre</guava.version>
+ <hadoop.version>3.0.0-beta1</hadoop.version>
+ <junit.version>4.11</junit.version>
+ <slf4j.version>1.7.10</slf4j.version>
diff --git a/testdata/cluster/hive/patch2-HIVE-20038.diff b/testdata/cluster/hive/patch2-HIVE-20038.diff
new file mode 100644
index 000000000..0025a64e3
--- /dev/null
+++ b/testdata/cluster/hive/patch2-HIVE-20038.diff
@@ -0,0 +1,67 @@
+diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+index 21f8268..949a9e8 100644
+--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
++++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+@@ -224,7 +224,9 @@ public void closeWriters(boolean abort) throws HiveException {
+ private void commit(FileSystem fs, List<Path> commitPaths) throws HiveException {
+ for (int idx = 0; idx < outPaths.length; ++idx) {
+ try {
+- commitOneOutPath(idx, fs, commitPaths);
++ if (outPaths[idx] != null) {
++ commitOneOutPath(idx, fs, commitPaths);
++ }
+ } catch (IOException e) {
+ throw new HiveException("Unable to commit output from: " +
+ outPaths[idx] + " to: " + finalPaths[idx], e);
+diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+index cf68d32..bbe9d5a 100644
+--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
++++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+@@ -184,6 +184,47 @@ public void testNoBuckets() throws Exception {
+ assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
+ }
+
++ @Test
++ public void testNoBucketsDP() throws Exception {
++ hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
++ int[][] sourceVals1 = {{0,0,0},{3,3,3}};
++ int[][] sourceVals2 = {{1,1,1},{2,2,2}};
++ int[][] sourceVals3 = {{3,3,3},{4,4,4}};
++ int[][] sourceVals4 = {{5,5,5},{6,6,6}};
++ runStatementOnDriver("drop table if exists tmp");
++ runStatementOnDriver("create table tmp (c1 integer, c2 integer) partitioned by (c3 integer) stored as orc " +
++ "tblproperties('transactional'='false')");
++ runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1));
++ runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2));
++ runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals3));
++ runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals4));
++ runStatementOnDriver("drop table if exists nobuckets");
++ runStatementOnDriver("create table nobuckets (c1 integer, c2 integer) partitioned by (c3 integer) stored " +
++ "as orc tblproperties('transactional'='true', 'transactional_properties'='default')");
++ String stmt = "insert into nobuckets partition(c3) select * from tmp";
++ runStatementOnDriver(stmt);
++ List<String> rs = runStatementOnDriver(
++ "select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by ROW__ID");
++ Assert.assertEquals("", 8, rs.size());
++ LOG.warn("after insert");
++ for(String s : rs) {
++ LOG.warn(s);
++ }
++
++ rs = runStatementOnDriver(
++ "select * from nobuckets where c2 in (0,3)");
++ Assert.assertEquals(3, rs.size());
++ runStatementOnDriver("update nobuckets set c2 = 17 where c2 in(0,3)");
++ rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by INPUT__FILE__NAME, ROW__ID");
++ LOG.warn("after update");
++ for(String s : rs) {
++ LOG.warn(s);
++ }
++ rs = runStatementOnDriver(
++ "select * from nobuckets where c2=17");
++ Assert.assertEquals(3, rs.size());
++ }
++
+ /**
+ * See CTAS tests in TestAcidOnTez
+ */
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index dca6deaaa..37e42652d 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -156,6 +156,14 @@ def parse_test_file(test_file_name, valid_section_names, skip_unknown_sections=T
with open(test_file_name, 'rb') as test_file:
file_data = test_file.read()
if encoding: file_data = file_data.decode(encoding)
+ if os.environ["USE_APACHE_HIVE"] == "true":
+ # Remove Hive 4.0 feature for tpcds_schema_template.sql
+ if "tpcds_schema_template" in test_file_name:
+ # HIVE-20703
+ file_data = file_data.replace(
+ 'set hive.optimize.sort.dynamic.partition.threshold=1;', '')
+ # HIVE-18284
+ file_data = file_data.replace('distribute by ss_sold_date_sk', '')
return parse_test_file_text(file_data, valid_section_names,
skip_unknown_sections)