You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/21 01:28:53 UTC
[20/25] incubator-kylin git commit: KYLIN-644 join table as null
error when build the cube
KYLIN-644 join table as null error when build the cube
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/53eb20fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/53eb20fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/53eb20fa
Branch: refs/heads/master
Commit: 53eb20faecb2fa071cd8f361f06e0cc99bab32b8
Parents: 933aa9e
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue Mar 17 14:57:16 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue Mar 17 14:57:16 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/JoinedFlatTable.java | 46 ++++++++++++++------
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 29 ------------
.../job/hadoop/hive/IIJoinedFlatTableDesc.java | 20 ---------
.../job/hadoop/hive/IJoinedFlatTableDesc.java | 5 ---
4 files changed, 33 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/53eb20fa/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index d9b1f66..7e714d0 100644
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -21,15 +21,13 @@ package org.apache.kylin.job;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.TimeZone;
+import java.util.*;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import org.apache.kylin.cube.model.DimensionDesc;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
@@ -54,6 +52,10 @@ import org.apache.kylin.metadata.model.TblColRef;
public class JoinedFlatTable {
+ public static final String FACT_TABLE_ALIAS = "FACT_TABLE";
+
+ public static final String LOOKUP_TABLE_ALAIS_PREFIX = "LOOKUP_";
+
public static String getTableDir(IJoinedFlatTableDesc intermediateTableDesc, String storageDfsDir, String jobUUID) {
return storageDfsDir + "/" + intermediateTableDesc.getTableName(jobUUID);
}
@@ -128,25 +130,43 @@ public class JoinedFlatTable {
StringBuilder sql = new StringBuilder();
sql.append("SELECT" + "\n");
String tableAlias;
+ Map<String, String> tableAliasMap = buildTableAliasMap(intermediateTableDesc.getDataModel());
for (int i = 0; i < intermediateTableDesc.getColumnList().size(); i++) {
IntermediateColumnDesc col = intermediateTableDesc.getColumnList().get(i);
if (i > 0) {
sql.append(",");
}
- tableAlias = intermediateTableDesc.getTableAlias(col.getTableName());
+ tableAlias = tableAliasMap.get(col.getTableName());
sql.append(tableAlias + "." + col.getColumnName() + "\n");
}
- appendJoinStatement(intermediateTableDesc, sql);
- appendWhereStatement(intermediateTableDesc, sql);
+ appendJoinStatement(intermediateTableDesc, sql, tableAliasMap);
+ appendWhereStatement(intermediateTableDesc, sql, tableAliasMap);
return sql.toString();
}
- private static void appendJoinStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql) {
+ private static Map buildTableAliasMap(DataModelDesc dataModelDesc) {
+ Map<String, String> tableAliasMap = new HashMap<String, String>();
+
+ tableAliasMap.put(dataModelDesc.getFactTable().toUpperCase(), FACT_TABLE_ALIAS);
+
+ int i = 1;
+ for (LookupDesc lookupDesc: dataModelDesc.getLookups()) {
+ JoinDesc join = lookupDesc.getJoin();
+ if (join != null) {
+ tableAliasMap.put(lookupDesc.getTable().toUpperCase(), LOOKUP_TABLE_ALAIS_PREFIX + i);
+ i++;
+ }
+
+ }
+ return tableAliasMap;
+ }
+
+ private static void appendJoinStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
Set<String> dimTableCache = new HashSet<String>();
DataModelDesc dataModelDesc = intermediateTableDesc.getDataModel();
String factTableName = dataModelDesc.getFactTable();
- String factTableAlias = intermediateTableDesc.getTableAlias(factTableName);
+ String factTableAlias = tableAliasMap.get(factTableName);
sql.append("FROM " + factTableName + " as " + factTableAlias + " \n");
for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
@@ -160,13 +180,13 @@ public class JoinedFlatTable {
if (pk.length != fk.length) {
throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
}
- sql.append(joinType + " JOIN " + dimTableName + " as " + intermediateTableDesc.getTableAlias(dimTableName) + "\n");
+ sql.append(joinType + " JOIN " + dimTableName + " as " + tableAliasMap.get(dimTableName) + "\n");
sql.append("ON ");
for (int i = 0; i < pk.length; i++) {
if (i > 0) {
sql.append(" AND ");
}
- sql.append(factTableAlias + "." + fk[i].getName() + " = " + intermediateTableDesc.getTableAlias(dimTableName) + "." + pk[i].getName());
+ sql.append(factTableAlias + "." + fk[i].getName() + " = " + tableAliasMap.get(dimTableName) + "." + pk[i].getName());
}
sql.append("\n");
@@ -176,7 +196,7 @@ public class JoinedFlatTable {
}
}
- private static void appendWhereStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql) {
+ private static void appendWhereStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
if (!(intermediateTableDesc instanceof CubeJoinedFlatTableDesc)) {
return;//TODO: for now only cube segments support filter and partition
}
@@ -207,7 +227,7 @@ public class JoinedFlatTable {
if (indexOfDot > 0) {
String partitionTableName = partitionColumnName.substring(0, indexOfDot);
String columeOnly = partitionColumnName.substring(indexOfDot);
- String partitionTableAlias = desc.getTableAlias(partitionTableName);
+ String partitionTableAlias = tableAliasMap.get(partitionTableName);
partitionColumnName = partitionTableAlias + columeOnly;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/53eb20fa/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
index 77475ec..a972692 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
@@ -44,14 +44,8 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
private int[] rowKeyColumnIndexes; // the column index on flat table
private int[][] measureColumnIndexes; // [i] is the i.th measure related column index on flat table
- // Map for table alias:
- // key -> table name;
- // value -> alias;
- private Map<String, String> tableAliasMap;
private List<IntermediateColumnDesc> columnList = Lists.newArrayList();
-
-
public CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment) {
this.cubeDesc = cubeDesc;
this.cubeSegment = cubeSegment;
@@ -120,24 +114,6 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
}
columnCount = columnIndex;
-
- buildTableAliasMap();
- }
-
- private void buildTableAliasMap() {
- tableAliasMap = new HashMap<String, String>();
-
- tableAliasMap.put(cubeDesc.getFactTable().toUpperCase(), FACT_TABLE_ALIAS);
-
- int i = 1;
- for (DimensionDesc dim : cubeDesc.getDimensions()) {
- JoinDesc join = dim.getJoin();
- if (join != null) {
- tableAliasMap.put(dim.getTable().toUpperCase(), LOOKUP_TABLE_ALAIS_PREFIX + i);
- i++;
- }
-
- }
}
private int contains(List<IntermediateColumnDesc> columnList, TblColRef c) {
@@ -191,11 +167,6 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
return cubeDesc.getModel().getCapacity();
}
- @Override
- public String getTableAlias(String tableName) {
- return tableAliasMap.get(tableName.toUpperCase());
- }
-
private static String colName(String canonicalColName) {
return canonicalColName.replace(".", "_");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/53eb20fa/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
index efc8cce..187b3da 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
@@ -53,22 +53,6 @@ public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc {
columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col));
columnIndex++;
}
- buildTableAliasMap();
- }
-
- private void buildTableAliasMap() {
- tableAliasMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- tableAliasMap.put(iiDesc.getFactTableName(), FACT_TABLE_ALIAS);
-
- int i = 1;
- for (LookupDesc lookupDesc : iiDesc.getModel().getLookups()) {
- JoinDesc join = lookupDesc.getJoin();
- if (join != null) {
- tableAliasMap.put(lookupDesc.getTable(), LOOKUP_TABLE_ALAIS_PREFIX + i);
- i++;
- }
- }
}
@Override
@@ -90,8 +74,4 @@ public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc {
return DataModelDesc.RealizationCapacity.SMALL;
}
- @Override
- public String getTableAlias(String tableName) {
- return tableAliasMap.get(tableName);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/53eb20fa/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
index 031f872..0841350 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
@@ -27,10 +27,6 @@ import org.apache.kylin.metadata.model.DataModelDesc;
*/
public interface IJoinedFlatTableDesc {
- public static final String FACT_TABLE_ALIAS = "FACT_TABLE";
-
- public static final String LOOKUP_TABLE_ALAIS_PREFIX = "LOOKUP_";
-
public String getTableName(String jobUUID);
public List<IntermediateColumnDesc> getColumnList();
@@ -39,5 +35,4 @@ public interface IJoinedFlatTableDesc {
public DataModelDesc.RealizationCapacity getCapacity();
- public String getTableAlias(String tableName);
}