You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/07 04:32:42 UTC
[43/50] [abbrv] kylin git commit: KYLIN-2250 HiveMRInput accepts
intermediate_view_table_name
KYLIN-2250 HiveMRInput accepts intermediate_view_table_name
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5c05880
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5c05880
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5c05880
Branch: refs/heads/master-cdh5.7
Commit: b5c058801b916ddc6e00085d652ad6a0497ce7d9
Parents: 7699973
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Dec 6 15:48:21 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Dec 6 16:46:40 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../apache/kylin/cube/cli/DictionaryGeneratorCLI.java | 2 +-
.../org/apache/kylin/metadata/model/TableDesc.java | 14 +++++++++-----
.../org/apache/kylin/source/hive/HiveMRInput.java | 13 +++++++++----
4 files changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0c80c07..bda1423 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -274,7 +274,7 @@ public class CubeManager implements IRealizationProvider {
SnapshotManager snapshotMgr = getSnapshotManager();
TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable));
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ if (tableDesc.isView()) {
String tableName = tableDesc.getMaterializedName();
tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable());
tableDesc.setName(tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 163c6ca..3e1ab0d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -98,7 +98,7 @@ public class DictionaryGeneratorCLI {
} else {
MetadataManager metadataManager = MetadataManager.getInstance(config);
TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ if (tableDesc.isView()) {
TableDesc materializedTbl = new TableDesc();
materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
materializedTbl.setName(tableDesc.getMaterializedName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 659de07..ab8c465 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,8 +26,8 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringSplitter;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Table Metadata from Source. All name should be uppercase.
@@ -36,7 +36,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class TableDesc extends RootPersistentEntity implements ISourceAware {
- public static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+ private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+ private static final String materializedTableNamePrefix = "kylin_intermediate_";
+
@JsonProperty("name")
private String name;
@JsonProperty("columns")
@@ -46,8 +48,6 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
@JsonProperty("table_type")
private String tableType;
- private static final String materializedTableNamePrefix = "kylin_intermediate_";
-
private DatabaseDesc database = new DatabaseDesc();
private String identity = null;
@@ -96,6 +96,10 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
}
return identity;
}
+
+ public boolean isView() {
+ return TABLE_TYPE_VIRTUAL_VIEW.equals(tableType);
+ }
public static String concatResourcePath(String tableIdentity) {
return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + ".json";
@@ -211,7 +215,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
return false;
if (!Arrays.equals(columns, tableDesc.columns))
return false;
-
+
return getIdentity().equals(tableDesc.getIdentity());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index aa603d7..fc2b982 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -60,6 +60,11 @@ import com.google.common.collect.Sets;
public class HiveMRInput implements IMRInput {
+ public static String getTableNameForHCat(TableDesc table) {
+ String tableName = table.isView() ? table.getMaterializedName() : table.getName();
+ return String.format("%s.%s", table.getDatabase(), tableName).toUpperCase();
+ }
+
@Override
public IMRBatchCubingInputSide getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
return new BatchCubingInputSide(flatDesc);
@@ -67,9 +72,9 @@ public class HiveMRInput implements IMRInput {
@Override
public IMRTableInputFormat getTableInputFormat(TableDesc table) {
- return new HiveTableInputFormat(table.getIdentity());
+ return new HiveTableInputFormat(getTableNameForHCat(table));
}
-
+
@Override
public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
return new IMRBatchMergeInputSide() {
@@ -167,7 +172,7 @@ public class HiveMRInput implements IMRInput {
for (JoinTableDesc lookupDesc : flatDesc.getDataModel().getJoinTables()) {
TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable());
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ if (tableDesc.isView()) {
lookupViewsTables.add(tableDesc);
}
}
@@ -180,7 +185,7 @@ public class HiveMRInput implements IMRInput {
hiveCmdBuilder.addStatement(useDatabaseHql);
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
for (TableDesc lookUpTableDesc : lookupViewsTables) {
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) {
+ if (lookUpTableDesc.isView()) {
StringBuilder createIntermediateTableHql = new StringBuilder();
createIntermediateTableHql.append("DROP TABLE IF EXISTS " + lookUpTableDesc.getMaterializedName() + ";\n");
createIntermediateTableHql.append("CREATE TABLE IF NOT EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");