[gobblin] branch master updated: [GOBBLIN-1502] dont add a partition if it already exists but the getPartition fails … (#3344)

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new e5576e7  [GOBBLIN-1502] dont add a partition if it already exists but the getPartition fails … (#3344)
e5576e7 is described below

commit e5576e72c2e57770164eb0796ba72524950c3a89
Author: Arjun Singh Bora <ab...@linkedin.com>
AuthorDate: Sat Aug 7 02:48:38 2021 +0530

    [GOBBLIN-1502] dont add a partition if it already exists but the getPartition fails … (#3344)
    
    * dont add a partition if it already exists but the getPartition fails with TException, dont alter a partition if a partition does not exist but the add_partition fails with TException
    
    * add a unit test to verify hive client behavior
---
 .../management/conversion/hive/HiveSourceTest.java | 38 ++++++++++++++++++++++
 .../hive/metastore/HiveMetaStoreBasedRegister.java |  4 +--
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
index 2351fd3..e6e4a72 100644
--- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
+++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
@@ -22,6 +22,8 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.joda.time.DateTime;
 import org.testng.Assert;
@@ -87,6 +89,42 @@ public class HiveSourceTest {
   }
 
   @Test
+  public void testAlreadyExistsPartition() throws Exception {
+    String dbName = "testdb";
+    String tableSdLoc = new File(this.tmpDir, TEST_TABLE_1).getAbsolutePath();
+
+    this.hiveMetastoreTestUtils.getLocalMetastoreClient().dropDatabase(dbName, false, true, true);
+
+    Table tbl = this.hiveMetastoreTestUtils.createTestAvroTable(dbName, TEST_TABLE_1, tableSdLoc, Optional.of("field"));
+
+    this.hiveMetastoreTestUtils.addTestPartition(tbl, ImmutableList.of("f1"), (int) System.currentTimeMillis());
+
+    try {
+      this.hiveMetastoreTestUtils.addTestPartition(tbl, ImmutableList.of("f1"), (int) System.currentTimeMillis());
+    } catch (AlreadyExistsException e) {
+      return;
+    }
+    Assert.fail();
+  }
+
+  @Test
+  public void testPartitionNotExists() throws Exception {
+    String dbName = "testdb1";
+    String tableSdLoc = new File(this.tmpDir, TEST_TABLE_1).getAbsolutePath();
+
+    this.hiveMetastoreTestUtils.getLocalMetastoreClient().dropDatabase(dbName, false, true, true);
+
+    Table tbl = this.hiveMetastoreTestUtils.createTestAvroTable(dbName, TEST_TABLE_1, tableSdLoc, Optional.of("field"));
+
+    try {
+      this.hiveMetastoreTestUtils.getLocalMetastoreClient().getPartition(tbl.getDbName(), tbl.getTableName(), "field");
+    } catch (NoSuchObjectException e) {
+      return;
+    }
+    Assert.fail();
+  }
+
+  @Test
   public void testGetWorkUnitsForPartitions() throws Exception {
     String dbName = "testdb3";
     String tableSdLoc = new File(this.tmpDir, TEST_TABLE_3).getAbsolutePath();
diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
index 1a8711e..d5b2393 100644
--- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
+++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
@@ -586,7 +586,7 @@ public class HiveMetaStoreBasedRegister extends HiveRegister {
         }
         log.info(String.format("Added partition %s to table %s with location %s", stringifyPartition(nativePartition),
             table.getTableName(), nativePartition.getSd().getLocation()));
-      } catch (TException e) {
+      } catch (AlreadyExistsException e) {
         try {
           if (this.skipDiffComputation) {
             onPartitionExistWithoutComputingDiff(table, nativePartition, e);
@@ -632,7 +632,7 @@ public class HiveMetaStoreBasedRegister extends HiveRegister {
             onPartitionExist(client, table, partition, nativePartition, existedPartition);
           }
         }
-      } catch (TException e) {
+      } catch (NoSuchObjectException e) {
         try (Timer.Context context = this.metricContext.timer(ADD_PARTITION_TIMER).time()) {
           client.add_partition(getPartitionWithCreateTimeNow(nativePartition));
         }