incubator-gobblin git commit: [GOBBLIN-174] fix distcp-ng, so it does not remove existing target files

[ab...@apache.org]

Repository: incubator-gobblin
Updated Branches:
  refs/heads/master 467fe8fc8 -> cec91e610


[GOBBLIN-174] fix distcp-ng, so it does not remove existing target files

fixing fullPathdiff() so it does not remove
existing target files incase NO_DELETE delete
method is specified

clearing existingTargetTable if 'existing entity
policy' is set to 'update_table'

Closes #2008 from arjun4084346/distcpNgFixNoDelete


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/cec91e61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/cec91e61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/cec91e61

Branch: refs/heads/master
Commit: cec91e61015568c6afb86c60f27181b7cc346a77
Parents: 467fe8f
Author: Arjun <ab...@linkedin.com>
Authored: Fri Jul 28 19:31:22 2017 -0700
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Fri Jul 28 19:31:22 2017 -0700

----------------------------------------------------------------------
 .../gobblin/data/management/copy/hive/HiveCopyEntityHelper.java    | 2 +-
 .../data/management/copy/hive/UnpartitionedTableFileSet.java       | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/cec91e61/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
index 230a02a..7cbd7bd 100644
--- a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
+++ b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
@@ -619,7 +619,7 @@ public class HiveCopyEntityHelper {
       if (shouldCopy) {
         builder.copyFile(sourcePath);
       } else {
-        // if not copying, we want to keep the file in the target
+        // If not copying, we want to keep the file in the target
         // at the end of this loop, all files in targetExistingPaths will be marked for deletion, so remove this file
         targetExistingPaths.remove(newPath);
         desiredTargetExistingPaths.remove(newPath);

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/cec91e61/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
index 9d7291a..1481535 100644
--- a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
+++ b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
@@ -70,6 +70,7 @@ public class UnpartitionedTableFileSet extends HiveFileSet {
             // Update the location of files while keep the existing table entity.
             log.warn("Source table will not be deregistered while file locaiton has been changed, update source table's"
                 + " file location to" + this.helper.getTargetTable().getDataLocation());
+            existingTargetTable = Optional.absent();
             break ;
           case REPLACE_TABLE:
             // Required to de-register the original table.