You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/24 02:59:46 UTC
incubator-kylin git commit: KYLIN-875 fix CI, MergeCuboidJobTest
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 993e0645e -> 69ac856bf
KYLIN-875 fix CI, MergeCuboidJobTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/69ac856b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/69ac856b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/69ac856b
Branch: refs/heads/0.8
Commit: 69ac856bf9e7f7327a8984a7dc866b4741792b90
Parents: 993e064
Author: Yang Li <li...@apache.org>
Authored: Fri Jul 24 08:58:14 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Jul 24 08:58:14 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DictionaryManager.java | 53 ++++++-------------
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../engine/mr/steps/MergeDictionaryStep.java | 2 +-
engine-mr/src/test/resources/data/dict/DW_SITES | Bin 5020 -> 0 bytes
engine-mr/src/test/resources/dict/DW_SITES | Bin 0 -> 5020 bytes
7 files changed, 21 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index f7a407b..881c9b2 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -29,6 +29,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.apache.kylin.source.ReadableTable.TableSignature;
@@ -190,17 +191,24 @@ public class DictionaryManager {
logger.info("building dictionary for " + col);
- Object[] tmp = decideSourceData(model, dict, col, factTableValueProvider);
- String srcTable = (String) tmp[0];
- String srcCol = (String) tmp[1];
- int srcColIdx = (Integer) tmp[2];
- ReadableTable inpTable = (ReadableTable) tmp[3];
+ TblColRef srcCol = decideSourceData(model, dict, col);
+ String srcTable = srcCol.getTable();
+ String srcColName = srcCol.getName();
+ int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
+
+ ReadableTable inpTable;
+ if (model.isFactTable(srcTable)) {
+ inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
+ } else {
+ TableDesc tableDesc = MetadataManager.getInstance(config).getTableDesc(srcTable);
+ inpTable = TableSourceFactory.createReadableTable(tableDesc);
+ }
TableSignature inputSig = inpTable.getSignature();
if (inputSig == null) // table does not exists
return null;
- DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inputSig);
+ DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcColName, srcColIdx, col.getDatatype(), inputSig);
String dupDict = checkDupByInfo(dictInfo);
if (dupDict != null) {
@@ -215,28 +223,10 @@ public class DictionaryManager {
}
/**
- * Get column origin
- *
- * @return 1. source table name
- * 2. column name
- * 3. column cardinal in source table
- * 4. ReadableTable object
+ * Decide a dictionary's source data, leverage PK-FK relationship.
*/
-
- public Object[] decideSourceData(DataModelDesc model, String dict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
- String srcTable;
- String srcCol;
- int srcColIdx;
- ReadableTable table;
- MetadataManager metaMgr = MetadataManager.getInstance(config);
-
- // Decide source data of dictionary:
- // 1. If 'useDict' specifies pre-defined data set, use that
- // 2. Otherwise find a lookup table to scan through
-
+ public TblColRef decideSourceData(DataModelDesc model, String dict, TblColRef col) throws IOException {
// Note FK on fact table is supported by scan the related PK on lookup table
-
- // normal case, source from lookup table
if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) {
// FK on fact table and join type is inner, use PK from lookup instead
if (model.isFactTable(col.getTable())) {
@@ -244,19 +234,10 @@ public class DictionaryManager {
if (pkCol != null)
col = pkCol; // scan the counterparty PK on lookup table instead
}
- srcTable = col.getTable();
- srcCol = col.getName();
- srcColIdx = col.getColumnDesc().getZeroBasedIndex();
- if (model.isFactTable(col.getTable())) {
- table = (factTableValueProvider == null) ? null : factTableValueProvider.getDistinctValuesFor(col);
- } else {
- table = TableSourceFactory.createReadableTable(metaMgr.getTableDesc(col.getTable()));
- }
+ return col;
}
else
throw new IllegalArgumentException("Unknown dictionary value: " + dict);
-
- return new Object[] { srcTable, srcCol, srcColIdx, table };
}
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 592c45d..283c3a1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -62,7 +62,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
if (!rowKey.isUseDictionary(col))
continue;
- String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+ String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
if (cubeDesc.getModel().isFactTable(scanTable)) {
factDictCols.add(i);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 2281432..3bd8630 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -89,7 +89,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
else {
ret = cubeDesc.getRowkey().isUseDictionary(col);
if (ret) {
- String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+ String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
}
dictsNeedMerging.put(col, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index bf3a3a4..3d3b80d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -76,7 +76,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
else {
ret = cubeDesc.getRowkey().isUseDictionary(col);
if (ret) {
- String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+ String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
}
dictsNeedMerging.put(col, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
index d99cb03..7ca16d5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -108,7 +108,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
for (DimensionDesc dim : cubeDesc.getDimensions()) {
for (TblColRef col : dim.getColumnRefs()) {
if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
- String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+ String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
colsNeedMeringDict.add(col);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/engine-mr/src/test/resources/data/dict/DW_SITES
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/resources/data/dict/DW_SITES b/engine-mr/src/test/resources/data/dict/DW_SITES
deleted file mode 100644
index 6f6af46..0000000
Binary files a/engine-mr/src/test/resources/data/dict/DW_SITES and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69ac856b/engine-mr/src/test/resources/dict/DW_SITES
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/resources/dict/DW_SITES b/engine-mr/src/test/resources/dict/DW_SITES
new file mode 100644
index 0000000..6f6af46
Binary files /dev/null and b/engine-mr/src/test/resources/dict/DW_SITES differ