You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/02/15 05:36:21 UTC
[2/2] kylin git commit: code review: change parameter name from
isPartial to isFullTable
code review: change parameter name from isPartial to isFullTable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09377224
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09377224
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09377224
Branch: refs/heads/master
Commit: 09377224ce2e26fdd1d00b952d8f5bca7654bd07
Parents: 545d145
Author: shaofengshi <sh...@apache.org>
Authored: Wed Feb 15 13:36:17 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Feb 15 13:36:17 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/IMRInput.java | 3 ++-
.../main/java/org/apache/kylin/engine/mr/MRUtil.java | 14 ++++++++------
.../org/apache/kylin/source/hive/HiveMRInput.java | 10 +++++-----
.../hive/cardinality/ColumnCardinalityMapper.java | 2 +-
.../hive/cardinality/HiveColumnCardinalityJob.java | 2 +-
.../org/apache/kylin/source/kafka/KafkaMRInput.java | 10 +++++-----
6 files changed, 22 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
index f6578f9..10d4879 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
@@ -35,7 +35,8 @@ public interface IMRInput {
/** Return an InputFormat that reads from specified table. */
public IMRTableInputFormat getTableInputFormat(TableDesc table);
- public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isPartial);
+ /** Return an InputFormat that reads from specified table; "isFullTable" indicates whether it is a full table read or not. */
+ public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isFullTable);
/** Return a helper to participate in batch cubing merge job flow. */
public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg);
http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index eb7bc4b..5ba9424 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -43,20 +43,22 @@ public class MRUtil {
return SourceFactory.createEngineAdapter(seg, IMRInput.class).getBatchCubingInputSide(flatDesc);
}
+ @Deprecated
public static IMRTableInputFormat getTableInputFormat(String tableName) {
- return getTableInputFormat(getTableDesc(tableName));
+ return getTableInputFormat(tableName, true);
}
- public static IMRTableInputFormat getTableInputFormat(String tableName, boolean isPartial) {
- return getTableInputFormat(getTableDesc(tableName), isPartial);
+ public static IMRTableInputFormat getTableInputFormat(String tableName, boolean isFullTable) {
+ return getTableInputFormat(getTableDesc(tableName), isFullTable);
}
+ @Deprecated
public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc) {
- return SourceFactory.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc);
+ return getTableInputFormat(tableDesc, true);
}
- public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc, boolean isPartial) {
- return SourceFactory.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc, isPartial);
+ public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc, boolean isFullTable) {
+ return SourceFactory.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc, isFullTable);
}
private static TableDesc getTableDesc(String tableName) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index c1ea45a..39a0273 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -60,8 +60,8 @@ import com.google.common.collect.Sets;
public class HiveMRInput implements IMRInput {
- public static String getTableNameForHCat(TableDesc table, boolean isPartial) {
- String tableName = (table.isView() || isPartial) ? table.getMaterializedName() : table.getName();
+ public static String getTableNameForHCat(TableDesc table, boolean isFullTable) {
+ String tableName = (table.isView() || isFullTable == false) ? table.getMaterializedName() : table.getName();
return String.format("%s.%s", table.getDatabase(), tableName).toUpperCase();
}
@@ -72,12 +72,12 @@ public class HiveMRInput implements IMRInput {
@Override
public IMRTableInputFormat getTableInputFormat(TableDesc table) {
- return new HiveTableInputFormat(getTableNameForHCat(table, false));
+ return new HiveTableInputFormat(getTableNameForHCat(table, true));
}
@Override
- public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isPartial) {
- return new HiveTableInputFormat(getTableNameForHCat(table, isPartial));
+ public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isFullTable) {
+ return new HiveTableInputFormat(getTableNameForHCat(table, isFullTable));
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index 145ed31..c712605 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -62,7 +62,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
String tableName = conf.get(BatchConstants.CFG_TABLE_NAME);
tableDesc = MetadataManager.getInstance(config).getTableDesc(tableName);
- tableInputFormat = MRUtil.getTableInputFormat(tableDesc);
+ tableInputFormat = MRUtil.getTableInputFormat(tableDesc, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index f439ccb..a837681 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -87,7 +87,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
job.getConfiguration().set("mapreduce.output.fileoutputformat.compress", "false");
// Mapper
- IMRTableInputFormat tableInputFormat = MRUtil.getTableInputFormat(table);
+ IMRTableInputFormat tableInputFormat = MRUtil.getTableInputFormat(table, true);
tableInputFormat.configureJob(job);
job.setMapperClass(ColumnCardinalityMapper.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/09377224/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 2d5a301..c7b327f 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -74,6 +74,11 @@ public class KafkaMRInput implements IMRInput {
@Override
public IMRTableInputFormat getTableInputFormat(TableDesc table) {
+ return getTableInputFormat(table, true);
+ }
+
+ @Override
+ public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isFullTable) {
KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(KylinConfig.getInstanceFromEnv());
KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(table.getIdentity());
List<TblColRef> columns = Lists.transform(Arrays.asList(table.getColumns()), new Function<ColumnDesc, TblColRef>() {
@@ -88,11 +93,6 @@ public class KafkaMRInput implements IMRInput {
}
@Override
- public IMRTableInputFormat getTableInputFormat(TableDesc table, boolean isPartial) {
- return getTableInputFormat(table);
- }
-
- @Override
public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
return new KafkaMRBatchMergeInputSide((CubeSegment) seg);
}