You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/05 13:39:18 UTC
[3/8] kylin git commit: KYLIN-1875 Normalize table and column names
in JSON
KYLIN-1875 Normalize table and column names in JSON
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e0713d34
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e0713d34
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e0713d34
Branch: refs/heads/KYILN-1875-B
Commit: e0713d348649e4a9c8abd2c1d1f47e08929a7660
Parents: fc1e11a
Author: Li Yang <li...@apache.org>
Authored: Tue Nov 29 20:32:04 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Dec 5 21:38:48 2016 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 22 +++++++++---
.../org/apache/kylin/cube/model/CubeDesc.java | 1 +
.../apache/kylin/cube/model/DimensionDesc.java | 19 +++++-----
.../model/validation/rule/FunctionRule.java | 37 ++++----------------
.../kylin/metadata/model/DataModelDesc.java | 13 ++++++-
.../kylin/metadata/model/FunctionDesc.java | 5 +--
.../metadata/model/ModelDimensionDesc.java | 19 +++++++---
.../kylin/metadata/model/PartitionDesc.java | 4 +--
8 files changed, 64 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 677b713..011035b 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -81,8 +81,7 @@ public class FactTableGenerator {
// table(appear as fk in fact table)
TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- // possible values of lookupTableKeys, extracted from existing lookup
- // tables.
+ // possible values of lookupTableKeys, extracted from existing lookup tables.
// The key is in the format of tablename/columnname
TreeMap<String, ArrayList<String>> feasibleValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
@@ -244,7 +243,7 @@ public class FactTableGenerator {
JoinDesc join = dim.getJoin();
if (join != null) {
String lookupTable = dim.getTableRef().getTableIdentity();
- for (String column : join.getPrimaryKey()) {
+ for (String column : dropAlias(join.getPrimaryKey())) {
if (!lookupTableKeys.containsKey(lookupTable)) {
lookupTableKeys.put(lookupTable, new LinkedList<String>());
}
@@ -297,8 +296,8 @@ public class FactTableGenerator {
for (DimensionDesc dim : dimensions) {
JoinDesc jDesc = dim.getJoin();
if (jDesc != null) {
- String[] fks = jDesc.getForeignKey();
- String[] pks = jDesc.getPrimaryKey();
+ String[] fks = dropAlias(jDesc.getForeignKey());
+ String[] pks = dropAlias(jDesc.getPrimaryKey());
int num = fks.length;
for (int i = 0; i < num; ++i) {
String value = dim.getTableRef().getTableIdentity() + "/" + pks[i];
@@ -343,6 +342,19 @@ public class FactTableGenerator {
return createTable(this.rowCount, factTableCol2LookupCol, lookupCol2factTableCol, usedCols);
}
+ private String[] dropAlias(String[] aliasDotCol) {
+ String[] result = new String[aliasDotCol.length];
+ for (int i = 0; i < aliasDotCol.length; i++) {
+ String str = aliasDotCol[i];
+ int cut = str.lastIndexOf('.');
+ if (cut >= 0) {
+ str = str.substring(cut + 1);
+ }
+ result[i] = str;
+ }
+ return result;
+ }
+
private String normToTwoDigits(int v) {
if (v < 10)
return "0" + v;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 853571c..f95cceb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -543,6 +543,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
initMeasureColumns();
rowkey.init(this);
+
validateAggregationGroups(); // check if aggregation group is valid
for (AggregationGroup agg : this.aggregationGroups) {
agg.init(this, rowkey);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index 1106103..cd75228 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -21,7 +21,6 @@ package org.apache.kylin.cube.model;
import java.util.Arrays;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -54,16 +53,13 @@ public class DimensionDesc {
private TblColRef[] columnRefs;
public void init(CubeDesc cubeDesc) {
+ DataModelDesc model = cubeDesc.getModel();
+
if (name != null)
name = name.toUpperCase();
- if (table != null)
- table = table.toUpperCase();
-
- DataModelDesc model = cubeDesc.getModel();
- tableRef = model.findTable(this.getTable());
- if (tableRef == null)
- throw new IllegalStateException("Can't find table " + table + " for dimension " + name);
+ tableRef = model.findTable(table);
+ table = tableRef.getAlias();
join = null;
for (JoinTableDesc joinTable : model.getJoinTables()) {
@@ -73,11 +69,16 @@ public class DimensionDesc {
}
}
+ if (column != null && !"{FK}".equals(column)) {
+ column = model.findColumn(table, column).getName();
+ }
if (derived != null && derived.length == 0) {
derived = null;
}
if (derived != null) {
- StringUtil.toUpperCaseArray(derived, derived);
+ for (int i = 0; i < derived.length; i++) {
+ derived[i] = model.findColumn(table, derived[i]).getName();
+ }
}
if (derived != null && join == null) {
throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 1ed7325..36631ce 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -31,12 +31,10 @@ import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.measure.topn.TopNMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import com.google.common.collect.Lists;
@@ -153,35 +151,12 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
* @param value
*/
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
- String factTable = cube.getModel().getRootFactTable().getTableIdentity();
- if (StringUtils.isEmpty(factTable)) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be null.");
- return;
- }
- TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable);
- if (table == null) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube);
- return;
- }
- // Prepare column set
- Set<String> set = new HashSet<String>();
- ColumnDesc[] cdesc = table.getColumns();
- for (int i = 0; i < cdesc.length; i++) {
- ColumnDesc columnDesc = cdesc[i];
- set.add(columnDesc.getName());
- }
-
- String[] items = value.split(",");
- for (int i = 0; i < items.length; i++) {
- String item = items[i].trim();
- if (StringUtils.isEmpty(item)) {
- continue;
- }
- if (!set.contains(item)) {
- context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable);
- }
+ DataModelDesc model = cube.getModel();
+ try {
+ model.findColumn(value);
+ } catch (IllegalArgumentException e) {
+ context.addResult(ResultLevel.ERROR, e.getMessage());
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index d917571..898ff74 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -282,7 +282,7 @@ public class DataModelDesc extends RootPersistentEntity {
initTableAlias(tables);
initJoinColumns();
initJoinsTree();
- ModelDimensionDesc.capicalizeStrings(dimensions);
+ initDimensionsAndMetrics();
initPartitionDesc();
}
@@ -354,6 +354,15 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ private void initDimensionsAndMetrics() {
+ for (ModelDimensionDesc dim : dimensions) {
+ dim.init(this);
+ }
+ for (int i = 0; i < metrics.length; i++) {
+ metrics[i] = findColumn(metrics[i]).getIdentity();
+ }
+ }
+
private void initPartitionDesc() {
if (this.partitionDesc != null)
this.partitionDesc.init(this);
@@ -381,6 +390,7 @@ public class DataModelDesc extends RootPersistentEntity {
if (col == null || col.getTableRef().equals(dimTable) == false) {
throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity());
}
+ pks[i] = col.getIdentity();
pkCols[i] = col;
}
join.setPrimaryKeyColumns(pkCols);
@@ -393,6 +403,7 @@ public class DataModelDesc extends RootPersistentEntity {
if (col == null) {
throw new IllegalStateException("Can't find column " + fks[i] + " in table " + this.getRootFactTable());
}
+ fks[i] = col.getIdentity();
fkCols[i] = col;
}
join.setForeignKeyColumns(fkCols);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 7b41552..dfa6f3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -77,14 +77,11 @@ public class FunctionDesc {
expression = expression.toUpperCase();
returnDataType = DataType.getType(returnType);
- for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
- p.setValue(p.getValue().toUpperCase());
- }
-
ArrayList<TblColRef> colRefs = Lists.newArrayList();
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
if (p.isColumnType()) {
TblColRef colRef = model.findColumn(p.getValue());
+ p.setValue(colRef.getIdentity());
colRefs.add(colRef);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index d196155..6460f71 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -50,13 +50,24 @@ public class ModelDimensionDesc {
this.columns = columns;
}
+ void init(DataModelDesc model) {
+ table = table.toUpperCase();
+ if (columns != null) {
+ StringUtil.toUpperCaseArray(columns, columns);
+ }
+
+ if (model != null) {
+ table = model.findTable(table).getAlias();
+ for (int i = 0; i < columns.length; i++) {
+ columns[i] = model.findColumn(table, columns[i]).getName();
+ }
+ }
+ }
+
public static void capicalizeStrings(List<ModelDimensionDesc> dimensions) {
if (dimensions != null) {
for (ModelDimensionDesc modelDimensionDesc : dimensions) {
- modelDimensionDesc.setTable(modelDimensionDesc.getTable().toUpperCase());
- if (modelDimensionDesc.getColumns() != null) {
- StringUtil.toUpperCaseArray(modelDimensionDesc.getColumns(), modelDimensionDesc.getColumns());
- }
+ modelDimensionDesc.init(null);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 0261f41..9925990 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -66,11 +66,11 @@ public class PartitionDesc {
if (StringUtils.isEmpty(partitionDateColumn))
return;
- partitionDateColumn = partitionDateColumn.toUpperCase();
partitionDateColumnRef = model.findColumn(partitionDateColumn);
+ partitionDateColumn = partitionDateColumnRef.getIdentity();
if (StringUtils.isBlank(partitionTimeColumn) == false) {
- partitionTimeColumn = partitionTimeColumn.toUpperCase();
partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
+ partitionTimeColumn = partitionTimeColumnRef.getIdentity();
}
partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}