You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2017/10/25 09:13:40 UTC
incubator-gobblin git commit: [GOBBLIN-297] Changed access to
protected for HiveSource and Watermarkers
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 056888e02 -> f3eadceed
[GOBBLIN-297] Changed access to protected for HiveSource and Watermarkers
Closes #2151 from aditya1105/HiveSource
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/f3eadcee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/f3eadcee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/f3eadcee
Branch: refs/heads/master
Commit: f3eadceed0173f9b6b122f6e714531700d22941b
Parents: 056888e
Author: adsharma <ad...@linkedin.com>
Authored: Wed Oct 25 14:43:29 2017 +0530
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Wed Oct 25 14:43:29 2017 +0530
----------------------------------------------------------------------
.../conversion/hive/source/HiveSource.java | 32 ++++++++++----------
.../watermarker/PartitionLevelWatermarker.java | 14 ++++-----
.../hive/watermarker/TableLevelWatermarker.java | 2 +-
3 files changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/f3eadcee/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
index c72ee91..94d5ad9 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java
@@ -137,19 +137,19 @@ public class HiveSource implements Source {
public static final Gson GENERICS_AWARE_GSON = GsonInterfaceAdapter.getGson(Object.class);
public static final Splitter COMMA_BASED_SPLITTER = Splitter.on(",").omitEmptyStrings().trimResults();
- private MetricContext metricContext;
- private EventSubmitter eventSubmitter;
- private AvroSchemaManager avroSchemaManager;
-
- private HiveUnitUpdateProvider updateProvider;
- private HiveSourceWatermarker watermarker;
- private IterableDatasetFinder<HiveDataset> datasetFinder;
- private List<WorkUnit> workunits;
- private long maxLookBackTime;
- private long beginGetWorkunitsTime;
- private List<String> ignoreDataPathIdentifierList;
-
- private final ClassAliasResolver<HiveBaseExtractorFactory> classAliasResolver =
+ protected MetricContext metricContext;
+ protected EventSubmitter eventSubmitter;
+ protected AvroSchemaManager avroSchemaManager;
+
+ protected HiveUnitUpdateProvider updateProvider;
+ protected HiveSourceWatermarker watermarker;
+ protected IterableDatasetFinder<HiveDataset> datasetFinder;
+ protected List<WorkUnit> workunits;
+ protected long maxLookBackTime;
+ protected long beginGetWorkunitsTime;
+ protected List<String> ignoreDataPathIdentifierList;
+
+ protected final ClassAliasResolver<HiveBaseExtractorFactory> classAliasResolver =
new ClassAliasResolver<>(HiveBaseExtractorFactory.class);
@Override
@@ -219,7 +219,7 @@ public class HiveSource implements Source {
}
- private void createWorkunitForNonPartitionedTable(HiveDataset hiveDataset) throws IOException {
+ protected void createWorkunitForNonPartitionedTable(HiveDataset hiveDataset) throws IOException {
// Create workunits for tables
try {
@@ -280,7 +280,7 @@ public class HiveSource implements Source {
return hiveWorkUnit;
}
- private void createWorkunitsForPartitionedTable(HiveDataset hiveDataset, AutoReturnableObject<IMetaStoreClient> client) throws IOException {
+ protected void createWorkunitsForPartitionedTable(HiveDataset hiveDataset, AutoReturnableObject<IMetaStoreClient> client) throws IOException {
long tableProcessTime = new DateTime().getMillis();
this.watermarker.onTableProcessBegin(hiveDataset.getTable(), tableProcessTime);
@@ -425,7 +425,7 @@ public class HiveSource implements Source {
}
// Convert createTime from seconds to milliseconds
- private static long getCreateTime(Table table) {
+ protected static long getCreateTime(Table table) {
return TimeUnit.MILLISECONDS.convert(table.getTTable().getCreateTime(), TimeUnit.SECONDS);
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/f3eadcee/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java
index 4aacb26..5fd3f76 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java
@@ -112,31 +112,31 @@ public class PartitionLevelWatermarker implements HiveSourceWatermarker {
@Setter(AccessLevel.PACKAGE)
@VisibleForTesting
- private long leastWatermarkToPersistInState;
+ protected long leastWatermarkToPersistInState;
// Keep an additional 2 days of updates
- private static final int BUFFER_WATERMARK_DAYS_TO_PERSIST = 2;
+ protected static final int BUFFER_WATERMARK_DAYS_TO_PERSIST = 2;
/**
* Watermarks from previous state
*/
@Getter(AccessLevel.PACKAGE)
@VisibleForTesting
- private final TableWatermarks previousWatermarks;
+ protected final TableWatermarks previousWatermarks;
/**
* Current expected watermarks
*/
@Getter(AccessLevel.PACKAGE)
@VisibleForTesting
- private final TableWatermarks expectedHighWatermarks;
+ protected final TableWatermarks expectedHighWatermarks;
- private final HiveMetastoreClientPool pool;
+ protected final HiveMetastoreClientPool pool;
/**
* Delegates watermarking logic to {@link TableLevelWatermarker} for Non partitioned tables
*/
- private final TableLevelWatermarker tableLevelWatermarker;
+ protected final TableLevelWatermarker tableLevelWatermarker;
- private final HiveUnitUpdateProvider updateProvider;
+ protected final HiveUnitUpdateProvider updateProvider;
/**
* Reads and initialized the previous high watermarks from {@link SourceState#getPreviousDatasetStatesByUrns()}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/f3eadcee/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java
----------------------------------------------------------------------
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java
index c325d69..716d296 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java
@@ -53,7 +53,7 @@ public class TableLevelWatermarker implements HiveSourceWatermarker {
public static final Gson GSON = new Gson();
// Table complete name db@tb - list of previous workunitState
- private Map<String, LongWatermark> tableWatermarks;
+ protected Map<String, LongWatermark> tableWatermarks;
public TableLevelWatermarker(State state) {
this.tableWatermarks = Maps.newHashMap();