You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2018/02/20 18:36:39 UTC
incubator-gobblin git commit: [GOBBLIN-410] Support
REPLACE_TABLE_AND_PARTITIONS for Hive copies.
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 4f5223028 -> 11182dcf3
[GOBBLIN-410] Support REPLACE_TABLE_AND_PARTITIONS for Hive copies.
Closes #2287 from ibuenros/replace-table-and-
partitions
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/11182dcf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/11182dcf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/11182dcf
Branch: refs/heads/master
Commit: 11182dcf3ed771a38fbbaa350cb115cf5e0fe3e3
Parents: 4f52230
Author: ibuenros <is...@gmail.com>
Authored: Tue Feb 20 10:35:59 2018 -0800
Committer: Hung Tran <hu...@linkedin.com>
Committed: Tue Feb 20 10:35:59 2018 -0800
----------------------------------------------------------------------
.../gobblin/data/management/copy/hive/HiveCopyEntityHelper.java | 2 ++
.../gobblin/data/management/copy/hive/HivePartitionFileSet.java | 3 ++-
.../data/management/copy/hive/UnpartitionedTableFileSet.java | 1 +
3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/11182dcf/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
index 2580775..2b1a142 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java
@@ -200,6 +200,8 @@ public class HiveCopyEntityHelper {
REPLACE_PARTITIONS,
/** Deregister target table, do NOT delete its files, and create a new table with correct values. */
REPLACE_TABLE,
+ /** A combination of {@link #REPLACE_TABLE} and {@link #REPLACE_PARTITIONS}*/
+ REPLACE_TABLE_AND_PARTITIONS,
/** Keep the target table as registered while updating the file location */
UPDATE_TABLE,
/** Abort copying of conflict table. */
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/11182dcf/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java
index 34b6933..2c3817e 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java
@@ -96,7 +96,8 @@ public class HivePartitionFileSet extends HiveFileSet {
try {
checkPartitionCompatibility(targetPartition, this.existingTargetPartition.get());
} catch (IOException ioe) {
- if (hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_PARTITIONS) {
+ if (hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_PARTITIONS &&
+ hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_TABLE_AND_PARTITIONS) {
log.error("Source and target partitions are not compatible. Aborting copy of partition " + this.partition,
ioe);
return Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/11182dcf/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
index 21813fb..756b4dd 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
@@ -73,6 +73,7 @@ public class UnpartitionedTableFileSet extends HiveFileSet {
existingTargetTable = Optional.absent();
break ;
case REPLACE_TABLE:
+ case REPLACE_TABLE_AND_PARTITIONS:
// Required to de-register the original table.
log.warn("Source and target table are not compatible. Will override target table " + existingTargetTable.get()
.getDataLocation());