[gobblin] branch master updated: Fix bug where partitioned tables would always return the wrong equality in paths (#3472)

[zi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 316ab18  Fix bug where partitioned tables would always return the wrong equality in paths (#3472)
316ab18 is described below

commit 316ab18a4b157c9da1a3e8588e7b2b9c3dafe421
Author: William Lo <lo...@gmail.com>
AuthorDate: Thu Feb 24 14:35:51 2022 -0800

    Fix bug where partitioned tables would always return the wrong equality in paths (#3472)
---
 .../management/copy/hive/HiveCopyEntityHelper.java |  5 ++--
 .../copy/hive/HiveCopyEntityHelperTest.java        | 28 ++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
index a1ce36c..ae53035 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
@@ -748,10 +748,11 @@ public class HiveCopyEntityHelper {
     return builder.build();
   }
 
-  private void checkPartitionedTableCompatibility(Table desiredTargetTable, Table existingTargetTable)
+  @VisibleForTesting
+  protected void checkPartitionedTableCompatibility(Table desiredTargetTable, Table existingTargetTable)
       throws IOException {
 
-    if (HiveUtils.areTablePathsEquivalent(this.targetFs, desiredTargetTable.getDataLocation(), existingTargetTable.getDataLocation())) {
+    if (!HiveUtils.areTablePathsEquivalent(getTargetFs(), desiredTargetTable.getDataLocation(), existingTargetTable.getDataLocation())) {
       throw new HiveTableLocationNotMatchException(desiredTargetTable.getDataLocation(), existingTargetTable.getDataLocation());
     }
 
diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelperTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelperTest.java
index 5303c40..2bc4a63 100644
--- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelperTest.java
+++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelperTest.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.mockito.Mockito;
@@ -432,6 +433,33 @@ public class HiveCopyEntityHelperTest {
     Assert.assertEquals(helper.getDataset().getDatasetPath(), "/targetPath/db/table");
   }
 
+  @Test
+  public void testPartitionedTableCompatibility() throws Exception {
+    FieldSchema partitionSchema = new FieldSchema("part", "string", "some comment");
+    List partitions = new ArrayList();
+    Path testPath = new Path("/testPath/db/table");
+    Path existingTablePath = new Path("/existing/testPath/db/table");
+    org.apache.hadoop.hive.ql.metadata.Table table = new org.apache.hadoop.hive.ql.metadata.Table("testDb","table1");
+    table.setDataLocation(testPath);
+    partitions.add(partitionSchema);
+    table.setPartCols(partitions);
+    org.apache.hadoop.hive.ql.metadata.Table existingTargetTable = new Table("testDb","table1");
+    existingTargetTable.setDataLocation(existingTablePath);
+    existingTargetTable.setPartCols(partitions);
+    HiveDataset hiveDataset = Mockito.mock(HiveDataset.class);
+    HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
+    Mockito.when(helper.getDataset()).thenReturn(hiveDataset);
+    Mockito.when(helper.getExistingTargetTable()).thenReturn(Optional.of(existingTargetTable));
+    Mockito.when(helper.getTargetTable()).thenReturn(table);
+    // Mock filesystem resolver
+    FileSystem mockFS = Mockito.mock(FileSystem.class);
+    Mockito.when(helper.getTargetFs()).thenReturn(mockFS);
+    Mockito.when(mockFS.resolvePath(Mockito.any())).thenReturn(new Path("hdfs://testPath/db/table"));
+
+    Mockito.doCallRealMethod().when(helper).checkPartitionedTableCompatibility(table, existingTargetTable);
+    helper.checkPartitionedTableCompatibility(table, existingTargetTable);
+  }
+
   private boolean containsPath(Collection<FileStatus> statuses, Path path) {
     for (FileStatus status : statuses) {
       if (status.getPath().equals(path)) {