You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2018/03/21 08:30:30 UTC
[12/50] incubator-gobblin git commit: [GOBBLIN-399] Refactor
HiveSource#shouldCreateWorkunit() to accept table as parameter
[GOBBLIN-399] Refactor HiveSource#shouldCreateWorkunit() to accept table as parameter
Closes #2273 from aditya1105/metadata
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/af68d7ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/af68d7ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/af68d7ed
Branch: refs/heads/0.12.0
Commit: af68d7edd788685279e6e0cf2ad0b6cbf8e7f2d0
Parents: d29b72f
Author: aditya1105 <ad...@linkedin.com>
Authored: Thu Feb 1 11:38:08 2018 -0800
Committer: Hung Tran <hu...@linkedin.com>
Committed: Thu Feb 1 11:38:08 2018 -0800
----------------------------------------------------------------------
.../data/management/conversion/hive/source/HiveSource.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af68d7ed/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
index 94d5ad9..3ad99fd 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
@@ -238,7 +238,7 @@ public class HiveSource implements Source {
return;
}
- if (shouldCreateWorkunit(getCreateTime(hiveDataset.getTable()), updateTime, lowWatermark)) {
+ if (shouldCreateWorkunit(hiveDataset.getTable(), lowWatermark)) {
log.info(String.format(
"Creating workunit for table %s as updateTime %s or createTime %s is greater than low watermark %s",
@@ -384,6 +384,13 @@ public class HiveSource implements Source {
return shouldCreateWorkunit(createTime, updateTime, lowWatermark);
}
+ protected boolean shouldCreateWorkunit(Table table, LongWatermark lowWatermark)
+ throws UpdateNotFoundException {
+ long updateTime = this.updateProvider.getUpdateTime(table);
+ long createTime = getCreateTime(table);
+ return shouldCreateWorkunit(createTime, updateTime, lowWatermark);
+ }
+
/**
* Check if workunit needs to be created. Returns <code>true</code> If the
* <code>updateTime</code> is greater than the <code>lowWatermark</code> and <code>maxLookBackTime</code>