You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/02 13:19:19 UTC
[26/27] kylin git commit: KYLIN-1875 fix PK-FK derive from each other
KYLIN-1875 fix PK-FK derive from each other
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8b955aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8b955aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8b955aa
Branch: refs/heads/KYLIN-1875
Commit: f8b955aac7458db7335b92c615871a15cdbadecb
Parents: e1d6f7f
Author: Yang Li <li...@apache.org>
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 2 21:02:01 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 8 +--
.../org/apache/kylin/cube/model/CubeDesc.java | 52 +++++++++++---------
.../storage/gtrecord/CubeTupleConverter.java | 2 +-
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../translate/DerivedFilterTranslator.java | 2 +-
.../apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../query/enumerator/LookupTableEnumerator.java | 2 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 2 +-
9 files changed, 41 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4ba29af..d08aff2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -55,6 +54,7 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
@@ -656,10 +656,10 @@ public class CubeManager implements IRealizationProvider {
cubeMap.removeLocal(cubeName);
}
- public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
+ public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
- String tableName = dim.getTableRef().getTableIdentity();
- String[] pkCols = dim.getJoin().getPrimaryKey();
+ String tableName = join.getPKSide().getTableIdentity();
+ String[] pkCols = join.getPrimaryKey();
String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
if (snapshotResPath == null)
throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
public static class DeriveInfo {
public DeriveType type;
- public DimensionDesc dimension;
+ public JoinDesc join;
public TblColRef[] columns;
public boolean isOneToOne; // only used when ref from derived to host
- DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) {
+ DeriveInfo(DeriveType type, JoinDesc join, TblColRef[] columns, boolean isOneToOne) {
this.type = type;
- this.dimension = dimension;
+ this.join = join;
this.columns = columns;
this.isOneToOne = isOneToOne;
}
@Override
public String toString() {
- return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
+ return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
}
}
@@ -742,27 +743,34 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
for (int i = 0; i < derivedNames.length; i++) {
derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
}
- initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+ initDerivedMap(dimColArray, DeriveType.LOOKUP, join, derivedCols, derivedExtra);
}
-
- // PK-FK derive the other side
+
if (join != null) {
- TblColRef[] fk = join.getForeignKeyColumns();
- TblColRef[] pk = join.getPrimaryKeyColumns();
-
- allColumns.addAll(Arrays.asList(fk));
- allColumns.addAll(Arrays.asList(pk));
- for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(dimColArray, fk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(new TblColRef[] { dimColArray[find] }, DeriveType.PK_FK, dim, new TblColRef[] { derivedCol }, null);
- }
+ allColumns.addAll(Arrays.asList(join.getForeignKeyColumns()));
+ allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
+ }
+ }
+
+ // PK-FK derive the other side
+ Set<TblColRef> realDimensions = new HashSet<>(listDimensionColumnsExcludingDerived(true));
+ for (JoinTableDesc joinTable : model.getJoinTables()) {
+ JoinDesc join = joinTable.getJoin();
+ int n = join.getForeignKeyColumns().length;
+ for (int i = 0; i < n; i++) {
+ TblColRef pk = join.getPrimaryKeyColumns()[i];
+ TblColRef fk = join.getForeignKeyColumns()[i];
+ if (realDimensions.contains(pk) && !realDimensions.contains(fk)) {
+ initDimensionColRef(fk);
+ initDerivedMap(new TblColRef[] { pk }, DeriveType.PK_FK, join, new TblColRef[] { fk }, null);
+ } else if (realDimensions.contains(fk) && !realDimensions.contains(pk)) {
+ initDimensionColRef(pk);
+ initDerivedMap(new TblColRef[] { fk }, DeriveType.PK_FK, join, new TblColRef[] { pk }, null);
}
}
}
}
-
+
private String[][] splitDerivedColumnAndExtra(String[] derived) {
String[] cols = new String[derived.length];
String[] extra = new String[derived.length];
@@ -780,7 +788,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return new String[][] { cols, extra };
}
- private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
+ private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, String[] extra) {
if (hostCols.length == 0 || derivedCols.length == 0)
throw new IllegalStateException("host/derived columns must not be empty");
@@ -803,12 +811,12 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
if (infoList == null) {
hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
}
- infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
+ infoList.add(new DeriveInfo(type, join, derivedCols, false));
for (int i = 0; i < derivedCols.length; i++) {
TblColRef derivedCol = derivedCols[i];
boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
- toHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
+ toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 7ec24b2..3159318 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -266,7 +266,7 @@ public class CubeTupleConverter {
case LOOKUP:
return new IDerivedColumnFiller() {
CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
- LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension);
+ LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
int[] derivedColIdx = initDerivedColIdx();
Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 9c74cca..31573d0 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -294,7 +294,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
CubeSegment seg = cubeInstance.getLatestReadySegment();
- LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
+ LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
TupleFilter translatedFilter = translated.getFirst();
boolean loosened = translated.getSecond();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 160338f..13c655c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -54,7 +54,7 @@ public class DerivedFilterTranslator {
TblColRef derivedCol = compf.getColumn();
TblColRef[] hostCols = hostInfo.columns;
- TblColRef[] pkCols = hostInfo.dimension.getJoin().getPrimaryKeyColumns();
+ TblColRef[] pkCols = hostInfo.join.getPrimaryKeyColumns();
if (hostInfo.type == DeriveType.PK_FK) {
assert hostCols.length == 1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 90324b5..6d91753 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
+//@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 1c2bc2d..28ee623 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -67,7 +67,7 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());
CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
- LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
+ LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim.getJoin());
this.allRows = table.getAllRows();
OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 02aa64a..75c3fd7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -368,7 +368,7 @@ public class CubeStorageQuery implements IStorageQuery {
DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
CubeSegment seg = cubeInstance.getLatestReadySegment();
- LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
+ LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
TupleFilter translatedFilter = translated.getFirst();
boolean loosened = translated.getSecond();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index d2378b9..64feff0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -220,7 +220,7 @@ public class CubeTupleConverter {
case LOOKUP:
return new IDerivedColumnFiller() {
CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
- LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension);
+ LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
int[] derivedColIdx = initDerivedColIdx();
Array<String> lookupKey = new Array<String>(new String[hostColIdx.length]);