[iceberg] branch master updated: Hive: Update test code for Hive4 (#1645)

[bl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 9ad06c0  Hive: Update test code for Hive4 (#1645)
9ad06c0 is described below

commit 9ad06c0184df1f5865ce39072fcded8367829ed4
Author: Marton Bod <ma...@gmail.com>
AuthorDate: Tue Oct 27 19:22:25 2020 +0100

    Hive: Update test code for Hive4 (#1645)
    
    Co-authored-by: Marton Bod <mb...@cloudera.com>
---
 .../test/java/org/apache/iceberg/hive/TestHiveMetastore.java | 12 ++++++++++++
 .../iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java   | 10 +++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
index d2aab50..83d3508 100644
--- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
+++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
@@ -68,6 +68,17 @@ public class TestHiveMetastore {
           .impl(RetryingHMSHandler.class, HiveConf.class, IHMSHandler.class, boolean.class)
           .buildStatic();
 
+  // Hive3 introduces background metastore tasks (MetastoreTaskThread) for performing various cleanup duties. These
+  // threads are scheduled and executed in a static thread pool (org.apache.hadoop.hive.metastore.ThreadPool).
+  // This thread pool is shut down normally as part of the JVM shutdown hook, but since we're creating and tearing down
+  // multiple metastore instances within the same JVM, we have to call this cleanup method manually, otherwise
+  // threads from our previous test suite will be stuck in the pool with stale config, and keep on being scheduled.
+  // This can lead to issues, e.g. accidental Persistence Manager closure by ScheduledQueryExecutionsMaintTask.
+  private static final DynMethods.StaticMethod METASTORE_THREADS_SHUTDOWN = DynMethods.builder("shutdown")
+          .impl("org.apache.hadoop.hive.metastore.ThreadPool")
+          .orNoop()
+          .buildStatic();
+
   private File hiveLocalDir;
   private HiveConf hiveConf;
   private ExecutorService executorService;
@@ -125,6 +136,7 @@ public class TestHiveMetastore {
     if (baseHandler != null) {
       baseHandler.shutdown();
     }
+    METASTORE_THREADS_SHUTDOWN.invoke();
   }
 
   public HiveConf hiveConf() {
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
index 63e32dd..b71dd1b 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
@@ -106,7 +106,7 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
 
   private static final Set<String> IGNORED_PARAMS =
       ImmutableSet.of("bucketing_version", StatsSetupConst.ROW_COUNT,
-          StatsSetupConst.RAW_DATA_SIZE, StatsSetupConst.TOTAL_SIZE, StatsSetupConst.NUM_FILES);
+          StatsSetupConst.RAW_DATA_SIZE, StatsSetupConst.TOTAL_SIZE, StatsSetupConst.NUM_FILES, "numFilesErasureCoded");
 
   private static final int METASTORE_POOL_SIZE = 15;
 
@@ -332,8 +332,12 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
 
       // Check if the files are removed
       FileSystem fs = Util.getFs(hmsTableLocation, shell.getHiveConf());
-      Assert.assertEquals(1, fs.listStatus(hmsTableLocation).length);
-      Assert.assertEquals(0, fs.listStatus(new Path(hmsTableLocation, "metadata")).length);
+      if (fs.exists(hmsTableLocation)) {
+        // if table directory has been deleted, we're good. This is the expected behavior in Hive4.
+        // if table directory exists, its contents should have been cleaned up, save for an empty metadata dir (Hive3).
+        Assert.assertEquals(1, fs.listStatus(hmsTableLocation).length);
+        Assert.assertEquals(0, fs.listStatus(new Path(hmsTableLocation, "metadata")).length);
+      }
     }
   }