You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/08/11 05:24:34 UTC
[1/8] kylin git commit: minor,
fix get model NPE when getting cube list.
Repository: kylin
Updated Branches:
refs/heads/2.1.x c962feb75 -> f52190b12
minor, fix get model NPE when getting cube list.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d936d34
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d936d34
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d936d34
Branch: refs/heads/2.1.x
Commit: 6d936d34935b13211cb485a1a35f1d4803288a34
Parents: 91c08e8
Author: Jiatao Tao <24...@qq.com>
Authored: Wed Aug 2 04:27:07 2017 -0500
Committer: GitHub <no...@github.com>
Committed: Wed Aug 2 04:27:07 2017 -0500
----------------------------------------------------------------------
.../rest/controller2/CubeControllerV2.java | 24 ++++++++++++++++----
1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d936d34/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 9ffc062..cdc09b9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -109,7 +109,11 @@ public class CubeControllerV2 extends BasicController {
// official cubes
for (CubeInstance cube : cubes) {
- response.add(createCubeInstanceResponse(cube));
+ try {
+ response.add(createCubeInstanceResponse(cube));
+ } catch (Exception e) {
+ logger.error("Error creating cube instance response, skipping.", e);
+ }
}
// draft cubes
@@ -174,9 +178,14 @@ public class CubeControllerV2 extends BasicController {
r.setModel(cube.getDescriptor().getModelName());
r.setPartitionDateStart(cube.getDescriptor().getPartitionDateStart());
- r.setPartitionDateColumn(cube.getModel().getPartitionDesc().getPartitionDateColumn());
- r.setIs_streaming(
- cube.getModel().getRootFactTable().getTableDesc().getSourceType() == ISourceAware.ID_STREAMING);
+ // cuz model doesn't have a state the label a model is broken,
+ // so in some case the model can not be loaded due to some check failed,
+ // but the cube in this model can still be loaded.
+ if (cube.getModel() != null) {
+ r.setPartitionDateColumn(cube.getModel().getPartitionDesc().getPartitionDateColumn());
+ r.setIs_streaming(
+ cube.getModel().getRootFactTable().getTableDesc().getSourceType() == ISourceAware.ID_STREAMING);
+ }
r.setProject(projectService.getProjectOfCube(cube.getName()));
return r;
@@ -202,7 +211,12 @@ public class CubeControllerV2 extends BasicController {
throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), cubeName));
}
- CubeInstanceResponse r = createCubeInstanceResponse(cube);
+ CubeInstanceResponse r;
+ try {
+ r = createCubeInstanceResponse(cube);
+ } catch (Exception e) {
+ throw new BadRequestException("Error getting cube instance response.", ResponseCode.CODE_UNDEFINED, e);
+ }
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, r, "");
}
[7/8] kylin git commit: KYLIN-2783 fix serialization
Posted by li...@apache.org.
KYLIN-2783 fix serialization
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1cae1f5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1cae1f5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1cae1f5b
Branch: refs/heads/2.1.x
Commit: 1cae1f5b7ab879f3ec0735e9d176a142556e4d23
Parents: cfda465
Author: Li Yang <li...@apache.org>
Authored: Fri Aug 11 10:47:04 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Aug 11 12:17:54 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/cuboid/DefaultCuboidScheduler.java | 5 +----
.../org/apache/kylin/engine/spark/SparkCubingByLayer.java | 10 +++-------
2 files changed, 4 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1cae1f5b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index 859ad20..b75acd5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube.cuboid;
-import java.io.Serializable;
-
/**
*/
@@ -45,8 +43,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-@SuppressWarnings("serial")
-public class DefaultCuboidScheduler extends CuboidScheduler implements Serializable {
+public class DefaultCuboidScheduler extends CuboidScheduler {
private final long max;
private final Set<Long> allCuboidIds;
private final Map<Long, List<Long>> parent2child;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1cae1f5b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 2f54e25..544e072 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -44,7 +44,6 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.common.RowKeySplitter;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
import org.apache.kylin.cube.model.CubeDesc;
@@ -169,7 +168,6 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
final Broadcast<CubeSegment> vCubeSegment = sc.broadcast(cubeSegment);
final NDCuboidBuilder ndCuboidBuilder = new NDCuboidBuilder(vCubeSegment.getValue(), new RowKeyEncoderProvider(vCubeSegment.getValue()));
- final Broadcast<CuboidScheduler> vCuboidScheduler = sc.broadcast(vCubeDesc.getValue().getCuboidScheduler());
final int measureNum = cubeDesc.getMeasures().size();
int countMeasureIndex = 0;
@@ -259,7 +257,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
saveToHDFS(allRDDs[0], vCubeDesc.getValue(), outputPath, 0, confOverwrite);
// aggregate to ND cuboids
- PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), vCuboidScheduler.getValue(), ndCuboidBuilder);
+ PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), ndCuboidBuilder);
for (level = 1; level <= totalLevels; level++) {
partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
@@ -342,15 +340,13 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
CubeSegment cubeSegment;
CubeDesc cubeDesc;
- CuboidScheduler cuboidScheduler;
NDCuboidBuilder ndCuboidBuilder;
RowKeySplitter rowKeySplitter;
transient boolean initialized = false;
- CuboidFlatMap(CubeSegment cubeSegment, CubeDesc cubeDesc, CuboidScheduler cuboidScheduler, NDCuboidBuilder ndCuboidBuilder) {
+ CuboidFlatMap(CubeSegment cubeSegment, CubeDesc cubeDesc, NDCuboidBuilder ndCuboidBuilder) {
this.cubeSegment = cubeSegment;
this.cubeDesc = cubeDesc;
- this.cuboidScheduler = cuboidScheduler;
this.ndCuboidBuilder = ndCuboidBuilder;
this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
}
@@ -366,7 +362,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
long cuboidId = rowKeySplitter.split(key);
Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
- Collection<Long> myChildren = cuboidScheduler.getSpanningCuboid(cuboidId);
+ Collection<Long> myChildren = cubeDesc.getCuboidScheduler().getSpanningCuboid(cuboidId);
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {
[6/8] kylin git commit: KYLIN-2633 Upgrade Spark to 2.1 kylin-it
change spark_2.10 to spark_2.11
Posted by li...@apache.org.
KYLIN-2633
Upgrade Spark to 2.1
kylin-it change spark_2.10 to spark_2.11
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfda465b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfda465b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfda465b
Branch: refs/heads/2.1.x
Commit: cfda465b4591d74d4b3e01b44d0616dab58d8fbe
Parents: 7971121
Author: yiming.xu <10...@qq.com>
Authored: Fri Aug 11 09:31:20 2017 +0800
Committer: YIMINGXU <10...@qq.com>
Committed: Thu Aug 10 22:01:54 2017 -0500
----------------------------------------------------------------------
kylin-it/pom.xml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfda465b/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 9583b42..4e581f8 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -253,7 +253,7 @@
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
+ <artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
@@ -266,7 +266,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
+ <artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
@@ -279,7 +279,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.10</artifactId>
+ <artifactId>spark-hive_2.11</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
[3/8] kylin git commit: minor,
logging derive translated filter columns
Posted by li...@apache.org.
minor, logging derive translated filter columns
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/02868adf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/02868adf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/02868adf
Branch: refs/heads/2.1.x
Commit: 02868adfc3f264fbb32ecc025110aa3d69e20657
Parents: 42abb73
Author: lidongsjtu <li...@apache.org>
Authored: Fri Aug 4 17:14:47 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Fri Aug 4 18:28:37 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../storage/gtrecord/GTCubeStorageQueryBase.java | 15 ++++++++++-----
2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/02868adf/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0beb926..4215746 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -677,7 +677,7 @@ public class CubeManager implements IRealizationProvider {
String[] pkCols = join.getPrimaryKey();
String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
if (snapshotResPath == null)
- throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment"
+ throw new IllegalStateException("No snapshot for table '" + tableName + "' found on cube segment"
+ cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/02868adf/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 22f5fc9..ea3811c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -140,10 +140,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
Set<TblColRef> filterColumnD = Sets.newHashSet();
TupleFilter filterD = translateDerived(filter, loosenedColumnD);
groupsD.addAll(loosenedColumnD);
- TupleFilter.collectColumns(filter, filterColumnD);
+ TupleFilter.collectColumns(filterD, filterColumnD);
// set limit push down
- enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context);
+ enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filterD, loosenedColumnD, sqlDigest.aggregations, context);
// set whether to aggregate results from multiple partitions
enableStreamAggregateIfBeneficial(cuboid, groupsD, context);
// set query deadline
@@ -316,9 +316,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return compf;
DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
- CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
- CubeSegment seg = cubeInstance.getLatestReadySegment();
- LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
+ LookupStringTable lookup = getLookupStringTableForDerived(derived, hostInfo);
Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
TupleFilter translatedFilter = translated.getFirst();
boolean loosened = translated.getSecond();
@@ -328,6 +326,13 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return translatedFilter;
}
+ @SuppressWarnings("unchecked")
+ protected LookupStringTable getLookupStringTableForDerived(TblColRef derived, DeriveInfo hostInfo) {
+ CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
+ CubeSegment seg = cubeInstance.getLatestReadySegment();
+ return cubeMgr.getLookupTable(seg, hostInfo.join);
+ }
+
private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
if (filter == null)
return;
[8/8] kylin git commit: Merge commit
'c962feb75605c1919b88ce5b435e814efab626de' into kap-2.4.x
Posted by li...@apache.org.
Merge commit 'c962feb75605c1919b88ce5b435e814efab626de' into kap-2.4.x
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f52190b1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f52190b1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f52190b1
Branch: refs/heads/2.1.x
Commit: f52190b12c929f16f8f9aa24aa384af9d88eb259
Parents: 1cae1f5 c962feb
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Aug 11 13:23:42 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Aug 11 13:23:42 2017 +0800
----------------------------------------------------------------------
.../acl/0928468a-9fab-4185-9a14-6f2e7c74823f | 27 ++++++++++++++++++++
.../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b | 27 ++++++++++++++++++++
.../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c91b | 24 +++++++++++++++++
examples/sample_cube/template/user/ADMIN | 19 ++++++++++++++
.../apache/kylin/rest/service/AclService.java | 4 +--
.../coprocessor/endpoint/CubeVisitService.java | 18 ++++++-------
webapp/app/partials/cubeDesigner/measures.html | 3 ++-
7 files changed, 109 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[2/8] kylin git commit: KYLIN-2777 Should throw error when sql
contains non-equi left join
Posted by li...@apache.org.
KYLIN-2777 Should throw error when sql contains non-equi left join
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42abb73b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42abb73b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42abb73b
Branch: refs/heads/2.1.x
Commit: 42abb73bbd734e52e6c8fd1d8c05eec737bbe2ba
Parents: 6d936d3
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Aug 3 15:19:23 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Thu Aug 3 15:36:32 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/42abb73b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
index c31d1d0..5cbc0f0 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
@@ -55,7 +55,7 @@ public class OLAPJoinRule extends ConverterRule {
if (!info.isEqui() && join.getJoinType() != JoinRelType.INNER) {
// EnumerableJoinRel only supports equi-join. We can put a filter on top
// if it is an inner join.
- return null;
+ throw new IllegalArgumentException("We only support equi left join, please check join conditions!");
}
RelOptCluster cluster = join.getCluster();
[5/8] kylin git commit: KYLIN-2783 refactor CuboidScheduler
Posted by li...@apache.org.
KYLIN-2783 refactor CuboidScheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79711218
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79711218
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79711218
Branch: refs/heads/2.1.x
Commit: 79711218f14c74eee127c911aaeeca513e6a8793
Parents: bf887a7
Author: Li Yang <li...@apache.org>
Authored: Thu Aug 10 16:19:25 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Aug 10 16:43:03 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../main/resources/kylin-defaults.properties | 2 +
.../org/apache/kylin/cube/cuboid/Cuboid.java | 2 +-
.../org/apache/kylin/cube/cuboid/CuboidCLI.java | 4 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 266 +++----------------
.../cube/cuboid/DefaultCuboidScheduler.java | 263 ++++++++++++++++++
.../cube/inmemcubing/InMemCubeBuilder.java | 4 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 36 +--
.../org/apache/kylin/cube/util/CubingUtils.java | 25 +-
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 34 +--
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../mr/steps/FactDistinctColumnsMapper.java | 2 +-
.../kylin/engine/mr/steps/NDCuboidMapper.java | 8 +-
.../engine/spark/KylinKryoRegistrator.java | 2 +-
.../apache/kylin/engine/spark/SparkCubing.java | 5 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 21 +-
16 files changed, 357 insertions(+), 323 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 179d61f..3a06571 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -329,6 +329,10 @@ abstract public class KylinConfigBase implements Serializable {
// CUBE
// ============================================================================
+ public String getCuboidScheduler() {
+ return getOptional("kylin.cube.cuboid-scheduler", "org.apache.kylin.cube.cuboid.DefaultCuboidScheduler");
+ }
+
public double getJobCuboidSizeRatio() {
return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index ee25637..7c421f9 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -145,6 +145,8 @@ kylin.engine.mr.uhc-reducer-count=1
### CUBE | DICTIONARY ###
+kylin.cube.cuboid-scheduler=org.apache.kylin.cube.cuboid.DefaultCuboidScheduler
+
# 'auto', 'inmem', 'layer' or 'random' for testing
kylin.cube.algorithm=layer
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 76cb511..b71608c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -135,7 +135,7 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
return onTreeCandi;
}
- return new CuboidScheduler(cubeDesc).getValidParent(onTreeCandi);
+ return cubeDesc.getCuboidScheduler().findBestMatchCuboid(onTreeCandi);
}
private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index e2ff97e..d657a43 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -44,7 +44,7 @@ public class CuboidCLI {
}
public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
- CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
+ CuboidScheduler scheduler = cubeDesc.getCuboidScheduler();
long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
Collection<Long> cuboidSet = new TreeSet<Long>();
cuboidSet.add(baseCuboid);
@@ -136,7 +136,7 @@ public class CuboidCLI {
int levels = cube.getBuildLevel();
int[] allLevelCounts = new int[levels + 1];
- CuboidScheduler scheduler = new CuboidScheduler(cube);
+ CuboidScheduler scheduler = cube.getCuboidScheduler();
LinkedList<Long> nextQueue = new LinkedList<Long>();
LinkedList<Long> currentQueue = new LinkedList<Long>();
long baseCuboid = Cuboid.getBaseCuboidId(cube);
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index def3f03..1d8f589 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,73 +18,54 @@
package org.apache.kylin.cube.cuboid;
-import java.io.Serializable;
-
-/**
- */
-
-import java.util.ArrayDeque;
-import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Queue;
import java.util.Set;
-import javax.annotation.Nullable;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.cube.model.CubeDesc;
import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-@SuppressWarnings("serial")
-public class CuboidScheduler implements Serializable {
- private final CubeDesc cubeDesc;
- private final long max;
- private List<List<Long>> cuboidsByLayer;
-
- public CuboidScheduler(CubeDesc cubeDesc) {
- this.cubeDesc = cubeDesc;
- int size = this.cubeDesc.getRowkey().getRowKeyColumns().length;
- this.max = (long) Math.pow(2, size) - 1;
- }
-
- public int getCuboidCount() {
- return cubeDesc.getAllCuboids().size();
- }
-
- public List<Long> getSpanningCuboid(long cuboid) {
- if (cuboid > max || cuboid < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
- }
- List<Long> spanning = cubeDesc.getParent2Child().get(cuboid);
- if (spanning == null) {
- return Collections.EMPTY_LIST;
+/**
+ * Defines a cuboid tree, rooted by the base cuboid. A parent cuboid generates its child cuboids.
+ */
+abstract public class CuboidScheduler {
+
+ public static CuboidScheduler getInstance(CubeDesc cubeDesc) {
+ String clzName = cubeDesc.getConfig().getCuboidScheduler();
+ try {
+ Class<? extends CuboidScheduler> clz = ClassUtil.forName(clzName, CuboidScheduler.class);
+ return clz.getConstructor(CubeDesc.class).newInstance(cubeDesc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
- return spanning;
}
-
- public int getCardinality(long cuboid) {
- if (cuboid > max || cuboid < 0) {
- throw new IllegalArgumentException("Cubiod " + cuboid + " is out of scope 0-" + max);
- }
-
- return Long.bitCount(cuboid);
+
+ // ============================================================================
+
+ final protected CubeDesc cubeDesc;
+
+ protected CuboidScheduler(CubeDesc cubeDesc) {
+ this.cubeDesc = cubeDesc;
}
- public List<Long> getAllCuboidIds() {
- return Lists.newArrayList(cubeDesc.getAllCuboids());
- }
+ /** Returns all cuboids on the tree. */
+ abstract public Set<Long> getAllCuboidIds();
+
+ /** Returns the number of all cuboids. */
+ abstract public int getCuboidCount();
+
+ /** Returns the child cuboids of a parent. */
+ abstract public List<Long> getSpanningCuboid(long parentCuboid);
+
+ /** Returns a cuboid on the tree that best matches the request cuboid. */
+ abstract public long findBestMatchCuboid(long requestCuboid);
+
+ // ============================================================================
+
+ private transient List<List<Long>> cuboidsByLayer;
/**
* Get cuboids by layer. It's built from pre-expanding tree.
@@ -120,177 +101,4 @@ public class CuboidScheduler implements Serializable {
return cuboidsByLayer;
}
- /**
- * Collect cuboid from bottom up, considering all factor including black list
- * Build tree steps:
- * 1. Build tree from bottom up considering dim capping
- * 2. Kick out blacked-out cuboids from the tree
- * 3. Make sure all the cuboids have proper "parent", if not add it to the tree.
- * Direct parent is not necessary, can jump *forward* steps to find in-direct parent.
- * For example, forward = 1, grandparent can also be the parent. Only if both parent
- * and grandparent are missing, add grandparent to the tree.
- * @return Cuboid collection
- */
- public Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp() {
- int forward = cubeDesc.getParentForward();
- KylinConfig config = cubeDesc.getConfig();
-
- Set<Long> cuboidHolder = new HashSet<>();
-
- // build tree structure
- Set<Long> children = getLowestCuboids();
- long maxCombination = config.getCubeAggrGroupMaxCombination() * 10;
- maxCombination = maxCombination < 0 ? Long.MAX_VALUE : maxCombination;
- while (!children.isEmpty()) {
- if (cuboidHolder.size() > maxCombination) {
- throw new IllegalStateException("Too many cuboids for the cube. Cuboid combination reached " + cuboidHolder.size() + " and limit is " + maxCombination + ". Abort calculation.");
- }
- cuboidHolder.addAll(children);
- children = getOnTreeParentsByLayer(children);
- }
- cuboidHolder.add(Cuboid.getBaseCuboidId(cubeDesc));
-
- // kick off blacked
- cuboidHolder = Sets.newHashSet(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() {
- @Override
- public boolean apply(@Nullable Long cuboidId) {
- return !cubeDesc.isBlackedCuboid(cuboidId);
- }
- }));
-
- // fill padding cuboids
- Map<Long, List<Long>> parent2Child = Maps.newHashMap();
- Queue<Long> cuboidScan = new ArrayDeque<>();
- cuboidScan.addAll(cuboidHolder);
- while (!cuboidScan.isEmpty()) {
- long current = cuboidScan.poll();
- long parent = getParentOnPromise(current, cuboidHolder, forward);
-
- if (parent > 0) {
- if (!cuboidHolder.contains(parent)) {
- cuboidScan.add(parent);
- cuboidHolder.add(parent);
- }
- if (parent2Child.containsKey(parent)) {
- parent2Child.get(parent).add(current);
- } else {
- parent2Child.put(parent, Lists.newArrayList(current));
- }
- }
- }
-
- return Pair.newPair(cuboidHolder, parent2Child);
- }
-
- private long getParentOnPromise(long child, Set<Long> coll, int forward) {
- long parent = getOnTreeParent(child);
- if (parent < 0) {
- return -1;
- }
-
- if (coll.contains(parent) || forward == 0) {
- return parent;
- }
-
- return getParentOnPromise(parent, coll, forward - 1);
- }
-
- /**
- * Get the parent cuboid really on the spanning tree.
- * @param child an on-tree cuboid
- * @return
- */
- public long getValidParent(long child) {
- long parent = getOnTreeParent(child);
- while (parent > 0) {
- if (cubeDesc.getAllCuboids().contains(parent)) {
- break;
- }
- parent = getOnTreeParent(parent);
- }
-
- if (parent <= 0) {
- throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
- }
- return parent;
- }
-
- private long getOnTreeParent(long child) {
- Collection<Long> candidates = getOnTreeParents(child);
- if (candidates == null || candidates.isEmpty()) {
- return -1;
- }
- return Collections.min(candidates, Cuboid.cuboidSelectComparator);
- }
-
- /**
- * Get all parent for children cuboids, considering dim cap.
- * @param children children cuboids
- * @return all parents cuboids
- */
- private Set<Long> getOnTreeParentsByLayer(Collection<Long> children) {
- Set<Long> parents = new HashSet<>();
- for (long child : children) {
- parents.addAll(getOnTreeParents(child));
- }
- parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>() {
- @Override
- public boolean apply(@Nullable Long cuboidId) {
- if (cuboidId == Cuboid.getBaseCuboidId(cubeDesc)) {
- return true;
- }
-
- for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
- if (agg.isOnTree(cuboidId) && agg.checkDimCap(cuboidId)) {
- return true;
- }
- }
-
- return false;
- }
- }));
- return parents;
- }
-
- /**
- * Get all *possible* parent for a child cuboid
- * @param child Child cuboid ID
- * @return all *possible* parent cuboids
- */
- private Set<Long> getOnTreeParents(long child) {
- List<AggregationGroup> aggrs = Lists.newArrayList();
- for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
- if (agg.isOnTree(child)) {
- aggrs.add(agg);
- }
- }
-
- return getOnTreeParents(child, aggrs);
- }
-
- /**
- * Get lowest (not Cube building level) Cuboids for every Agg group
- * @return lowest level cuboids
- */
- private Set<Long> getLowestCuboids() {
- return getOnTreeParents(0L, cubeDesc.getAggregationGroups());
- }
-
- private Set<Long> getOnTreeParents(long child, Collection<AggregationGroup> groups) {
- Set<Long> parentCandidate = new HashSet<>();
-
- if (child == Cuboid.getBaseCuboidId(cubeDesc)) {
- return parentCandidate;
- }
-
- for (AggregationGroup agg : groups) {
- if (child == agg.getPartialCubeFullMask()) {
- parentCandidate.add(Cuboid.getBaseCuboidId(cubeDesc));
- return parentCandidate;
- }
- parentCandidate.addAll(AggregationGroupScheduler.getOnTreeParents(child, agg));
- }
-
- return parentCandidate;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
new file mode 100644
index 0000000..859ad20
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import java.io.Serializable;
+
+/**
+ */
+
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+@SuppressWarnings("serial")
+public class DefaultCuboidScheduler extends CuboidScheduler implements Serializable {
+ private final long max;
+ private final Set<Long> allCuboidIds;
+ private final Map<Long, List<Long>> parent2child;
+
+
+ public DefaultCuboidScheduler(CubeDesc cubeDesc) {
+ super(cubeDesc);
+
+ int size = this.cubeDesc.getRowkey().getRowKeyColumns().length;
+ this.max = (long) Math.pow(2, size) - 1;
+
+ Pair<Set<Long>, Map<Long, List<Long>>> pair = buildTreeBottomUp();
+ this.allCuboidIds = pair.getFirst();
+ this.parent2child = pair.getSecond();
+ }
+
+ @Override
+ public int getCuboidCount() {
+ return allCuboidIds.size();
+ }
+
+ @Override
+ public List<Long> getSpanningCuboid(long cuboid) {
+ if (cuboid > max || cuboid < 0) {
+ throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
+ }
+
+ List<Long> spanning = parent2child.get(cuboid);
+ if (spanning == null) {
+ return Collections.EMPTY_LIST;
+ }
+ return spanning;
+ }
+
+ @Override
+ public Set<Long> getAllCuboidIds() {
+ return Sets.newHashSet(allCuboidIds);
+ }
+
+ /**
+ * Collect cuboid from bottom up, considering all factor including black list
+ * Build tree steps:
+ * 1. Build tree from bottom up considering dim capping
+ * 2. Kick out blacked-out cuboids from the tree
+ * 3. Make sure all the cuboids have proper "parent", if not add it to the tree.
+ * Direct parent is not necessary, can jump *forward* steps to find in-direct parent.
+ * For example, forward = 1, grandparent can also be the parent. Only if both parent
+ * and grandparent are missing, add grandparent to the tree.
+ * @return Cuboid collection
+ */
+ private Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp() {
+ int forward = cubeDesc.getParentForward();
+ KylinConfig config = cubeDesc.getConfig();
+
+ Set<Long> cuboidHolder = new HashSet<>();
+
+ // build tree structure
+ Set<Long> children = getLowestCuboids();
+ long maxCombination = config.getCubeAggrGroupMaxCombination() * 10;
+ maxCombination = maxCombination < 0 ? Long.MAX_VALUE : maxCombination;
+ while (!children.isEmpty()) {
+ if (cuboidHolder.size() > maxCombination) {
+ throw new IllegalStateException("Too many cuboids for the cube. Cuboid combination reached " + cuboidHolder.size() + " and limit is " + maxCombination + ". Abort calculation.");
+ }
+ cuboidHolder.addAll(children);
+ children = getOnTreeParentsByLayer(children);
+ }
+ cuboidHolder.add(Cuboid.getBaseCuboidId(cubeDesc));
+
+ // kick off blacked
+ cuboidHolder = Sets.newHashSet(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() {
+ @Override
+ public boolean apply(@Nullable Long cuboidId) {
+ return !cubeDesc.isBlackedCuboid(cuboidId);
+ }
+ }));
+
+ // fill padding cuboids
+ Map<Long, List<Long>> parent2Child = Maps.newHashMap();
+ Queue<Long> cuboidScan = new ArrayDeque<>();
+ cuboidScan.addAll(cuboidHolder);
+ while (!cuboidScan.isEmpty()) {
+ long current = cuboidScan.poll();
+ long parent = getParentOnPromise(current, cuboidHolder, forward);
+
+ if (parent > 0) {
+ if (!cuboidHolder.contains(parent)) {
+ cuboidScan.add(parent);
+ cuboidHolder.add(parent);
+ }
+ if (parent2Child.containsKey(parent)) {
+ parent2Child.get(parent).add(current);
+ } else {
+ parent2Child.put(parent, Lists.newArrayList(current));
+ }
+ }
+ }
+
+ return Pair.newPair(cuboidHolder, parent2Child);
+ }
+
+ private long getParentOnPromise(long child, Set<Long> coll, int forward) {
+ long parent = getOnTreeParent(child);
+ if (parent < 0) {
+ return -1;
+ }
+
+ if (coll.contains(parent) || forward == 0) {
+ return parent;
+ }
+
+ return getParentOnPromise(parent, coll, forward - 1);
+ }
+
+ /**
+ * Get the parent cuboid really on the spanning tree.
+ * @param child an on-tree cuboid
+ * @return
+ */
+ @Override
+ public long findBestMatchCuboid(long child) {
+ long parent = getOnTreeParent(child);
+ while (parent > 0) {
+ if (cubeDesc.getAllCuboids().contains(parent)) {
+ break;
+ }
+ parent = getOnTreeParent(parent);
+ }
+
+ if (parent <= 0) {
+ throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
+ }
+ return parent;
+ }
+
+ private long getOnTreeParent(long child) {
+ Collection<Long> candidates = getOnTreeParents(child);
+ if (candidates == null || candidates.isEmpty()) {
+ return -1;
+ }
+ return Collections.min(candidates, Cuboid.cuboidSelectComparator);
+ }
+
+ /**
+ * Get all parent for children cuboids, considering dim cap.
+ * @param children children cuboids
+ * @return all parents cuboids
+ */
+ private Set<Long> getOnTreeParentsByLayer(Collection<Long> children) {
+ Set<Long> parents = new HashSet<>();
+ for (long child : children) {
+ parents.addAll(getOnTreeParents(child));
+ }
+ parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>() {
+ @Override
+ public boolean apply(@Nullable Long cuboidId) {
+ if (cuboidId == Cuboid.getBaseCuboidId(cubeDesc)) {
+ return true;
+ }
+
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ if (agg.isOnTree(cuboidId) && agg.checkDimCap(cuboidId)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+ }));
+ return parents;
+ }
+
+ /**
+ * Get all *possible* parent for a child cuboid
+ * @param child Child cuboid ID
+ * @return all *possible* parent cuboids
+ */
+ private Set<Long> getOnTreeParents(long child) {
+ List<AggregationGroup> aggrs = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ if (agg.isOnTree(child)) {
+ aggrs.add(agg);
+ }
+ }
+
+ return getOnTreeParents(child, aggrs);
+ }
+
+ /**
+ * Get lowest (not Cube building level) Cuboids for every Agg group
+ * @return lowest level cuboids
+ */
+ private Set<Long> getLowestCuboids() {
+ return getOnTreeParents(0L, cubeDesc.getAggregationGroups());
+ }
+
+ private Set<Long> getOnTreeParents(long child, Collection<AggregationGroup> groups) {
+ Set<Long> parentCandidate = new HashSet<>();
+
+ if (child == Cuboid.getBaseCuboidId(cubeDesc)) {
+ return parentCandidate;
+ }
+
+ for (AggregationGroup agg : groups) {
+ if (child == agg.getPartialCubeFullMask()) {
+ parentCandidate.add(Cuboid.getBaseCuboidId(cubeDesc));
+ return parentCandidate;
+ }
+ parentCandidate.addAll(AggregationGroupScheduler.getOnTreeParents(child, agg));
+ }
+
+ return parentCandidate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index a26e948..93736c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -33,8 +33,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.gridtable.CubeGridTable;
@@ -92,7 +92,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
super(cubeDesc, flatDesc, dictionaryMap);
- this.cuboidScheduler = new CuboidScheduler(cubeDesc);
+ this.cuboidScheduler = cubeDesc.getCuboidScheduler();
this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
this.totalCuboidCount = cuboidScheduler.getCuboidCount();
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 56fc9fa..5d8503d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -188,10 +188,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonInclude(JsonInclude.Include.NON_NULL)
private int parentForward = 3;
- // allCuboids and parent2Child lazy built
- private Set<Long> allCuboids;
- private Map<Long, List<Long>> parent2Child;
- private byte[] cuboidTreeLock = new byte[0];
+ // cuboid scheduler lazy built
+ transient private CuboidScheduler cuboidScheduler = null;
public boolean isEnableSharding() {
//in the future may extend to other storage that is shard-able
@@ -465,7 +463,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
* @return
*/
public int getBuildLevel() {
- return new CuboidScheduler(this).getCuboidsByLayer().size() - 1;
+ return getCuboidScheduler().getCuboidsByLayer().size() - 1;
}
@Override
@@ -558,11 +556,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
hostToDerivedMap = Maps.newHashMap();
extendedColumnToHosts = Maps.newHashMap();
cuboidBlackSet = Sets.newHashSet();
-
- synchronized (cuboidTreeLock) {
- allCuboids = null;
- parent2Child = null;
- }
+ cuboidScheduler = null;
}
public void init(KylinConfig config) {
@@ -618,13 +612,15 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
amendAllColumns();
}
- private void buildCuboidTree() {
- synchronized (cuboidTreeLock) {
- if (allCuboids == null || parent2Child == null) {
- Pair<Set<Long>, Map<Long, List<Long>>> ret = new CuboidScheduler(this).buildTreeBottomUp();
- allCuboids = ret.getFirst();
- parent2Child = ret.getSecond();
+ public CuboidScheduler getCuboidScheduler() {
+ if (cuboidScheduler != null)
+ return cuboidScheduler;
+
+ synchronized (this) {
+ if (cuboidScheduler == null) {
+ cuboidScheduler = CuboidScheduler.getInstance(this);
}
+ return cuboidScheduler;
}
}
@@ -1146,13 +1142,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
public Set<Long> getAllCuboids() {
- buildCuboidTree();
- return allCuboids;
- }
-
- public Map<Long, List<Long>> getParent2Child() {
- buildCuboidTree();
- return parent2Child;
+ return getCuboidScheduler().getAllCuboidIds();
}
public int getParentForward() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index b1b6bce..1c3c395 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -24,14 +24,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import javax.annotation.Nullable;
-
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.dict.DictionaryGenerator;
@@ -45,9 +42,7 @@ import org.apache.kylin.source.IReadableTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
@@ -62,28 +57,10 @@ public class CubingUtils {
public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
- final List<Long> allCuboidIds = new CuboidScheduler(cubeDesc).getAllCuboidIds();
+ final Set<Long> allCuboidIds = cubeDesc.getCuboidScheduler().getAllCuboidIds();
final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMap();
- Lists.transform(allCuboidIds, new Function<Long, Integer[]>() {
- @Nullable
- @Override
- public Integer[] apply(@Nullable Long cuboidId) {
- Integer[] result = new Integer[Long.bitCount(cuboidId)];
-
- long mask = Long.highestOneBit(baseCuboidId);
- int position = 0;
- for (int i = 0; i < rowkeyLength; i++) {
- if ((mask & cuboidId) > 0) {
- result[position] = i;
- position++;
- }
- mask = mask >> 1;
- }
- return result;
- }
- });
final Map<Long, HLLCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
for (Long cuboidId : allCuboidIds) {
result.put(cuboidId, new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index e2a71db..58c0edb 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -128,7 +128,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testGetSpanningCuboid2() {
CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
+ CuboidScheduler scheduler = cube.getCuboidScheduler();
// generate 8d
System.out.println("Spanning for 8D Cuboids");
@@ -156,7 +156,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testGetSpanningCuboid1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
+ CuboidScheduler scheduler = cube.getCuboidScheduler();
// generate 7d
System.out.println("Spanning for 7D Cuboids");
@@ -183,18 +183,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
}
@Test
- public void testGetCardinality() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- assertEquals(0, scheduler.getCardinality(0));
- assertEquals(7, scheduler.getCardinality(127));
- assertEquals(1, scheduler.getCardinality(1));
- assertEquals(1, scheduler.getCardinality(8));
- assertEquals(6, scheduler.getCardinality(126));
- }
-
- @Test
public void testCuboidGeneration1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
@@ -240,7 +228,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -253,7 +241,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts2() {
CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -266,7 +254,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts3() {
CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -279,7 +267,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts4() {
CubeDesc cube = getTestKylinCubeWithSellerLeft();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -292,7 +280,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts5() {
CubeDesc cube = getStreamingCubeDesc();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -305,7 +293,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts6() {
CubeDesc cube = getCIInnerJoinCube();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -318,7 +306,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testLargeCube() {
CubeDesc cube = getFiftyDimCubeDesc();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
long start = System.currentTimeMillis();
System.out.println(cuboidScheduler.getCuboidCount());
System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms");
@@ -329,7 +317,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4)));
CubeDesc cube = getTwentyDimCubeDesc();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
cuboidScheduler.getCuboidCount();
twentyFile.renameTo(new File(twentyFile.getPath() + ".bad"));
}
@@ -344,7 +332,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
}
CubeDescManager.clearCache();
CubeDesc cube = getCubeDescManager().getCubeDesc("ut_large_dimension_number");
- CuboidScheduler scheduler = new CuboidScheduler(cube);
+ CuboidScheduler scheduler = cube.getCuboidScheduler();
Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index a372c5b..e160d27 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -81,7 +81,7 @@ public class CubeStatsReader {
public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
ResourceStore store = ResourceStore.getStore(kylinConfig);
- cuboidScheduler = new CuboidScheduler(cubeSegment.getCubeDesc());
+ cuboidScheduler = cubeSegment.getCubeDesc().getCuboidScheduler();
String statsKey = cubeSegment.getStatisticsResourcePath();
File tmpSeqFile = writeTmpSeqFile(store.getResource(statsKey).inputStream);
Reader reader = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 713b7f7..8281759 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -88,7 +88,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
if (collectStatistics) {
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
- cuboidScheduler = new CuboidScheduler(cubeDesc);
+ cuboidScheduler = cubeDesc.getCuboidScheduler();
nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
List<Long> cuboidIdList = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index b924edc..782ce72 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -18,6 +18,9 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.IOException;
+import java.util.Collection;
+
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ByteArray;
@@ -36,9 +39,6 @@ import org.apache.kylin.engine.mr.common.NDCuboidBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collection;
-
/**
* @author George Song (ysong1)
*
@@ -75,7 +75,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
cubeDesc = cube.getDescriptor();
ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
// initialize CubiodScheduler
- cuboidScheduler = new CuboidScheduler(cubeDesc);
+ cuboidScheduler = cubeDesc.getCuboidScheduler();
rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 1980343..ac56075 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -130,7 +130,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(org.apache.kylin.cube.CubeSegment.class);
kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class);
kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class);
- kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class);
+ kyroClasses.add(org.apache.kylin.cube.cuboid.DefaultCuboidScheduler.class);
kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class);
kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class);
kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 57fd315..a094cc2 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -29,6 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
@@ -252,8 +253,8 @@ public class SparkCubing extends AbstractApplication {
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
- CuboidScheduler cuboidScheduler = new CuboidScheduler(cubeDesc);
- List<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds();
+ CuboidScheduler cuboidScheduler = cubeDesc.getCuboidScheduler();
+ Set<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds();
final HashMap<Long, HLLCounter> zeroValue = Maps.newHashMap();
for (Long id : allCuboidIds) {
zeroValue.put(id, new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/79711218/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 587ff78..2f54e25 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -17,6 +17,15 @@
*/
package org.apache.kylin.engine.spark;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -65,16 +74,8 @@ import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.storage.StorageLevel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
+import scala.Tuple2;
/**
@@ -168,7 +169,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
final Broadcast<CubeSegment> vCubeSegment = sc.broadcast(cubeSegment);
final NDCuboidBuilder ndCuboidBuilder = new NDCuboidBuilder(vCubeSegment.getValue(), new RowKeyEncoderProvider(vCubeSegment.getValue()));
- final Broadcast<CuboidScheduler> vCuboidScheduler = sc.broadcast(new CuboidScheduler(vCubeDesc.getValue()));
+ final Broadcast<CuboidScheduler> vCuboidScheduler = sc.broadcast(vCubeDesc.getValue().getCuboidScheduler());
final int measureNum = cubeDesc.getMeasures().size();
int countMeasureIndex = 0;
[4/8] kylin git commit: minor,
remove unused hard-code BI transform code (#1889)
Posted by li...@apache.org.
minor, remove unused hard-code BI transform code (#1889)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf887a7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf887a7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf887a7a
Branch: refs/heads/2.1.x
Commit: bf887a7a8fea0a5a0791e85c6c998df221889863
Parents: 02868ad
Author: Jiatao Tao <24...@qq.com>
Authored: Wed Aug 9 00:59:17 2017 -0500
Committer: GitHub <no...@github.com>
Committed: Wed Aug 9 00:59:17 2017 -0500
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ClassUtil.java | 1 -
.../query/util/CognosParenthesesEscape.java | 106 --------
.../query/util/ConvertToComputedColumn.java | 248 -------------------
.../org/apache/kylin/query/util/QueryUtil.java | 2 +-
.../query/util/CognosParenthesesEscapeTest.java | 89 -------
.../query/util/ConvertToComputedColumnTest.java | 124 ----------
.../src/test/resources/query/cognos/query01.sql | 28 ---
.../resources/query/cognos/query01.sql.expected | 28 ---
.../src/test/resources/query/cognos/query02.sql | 27 --
.../resources/query/cognos/query02.sql.expected | 27 --
.../src/test/resources/query/cognos/query03.sql | 29 ---
.../resources/query/cognos/query03.sql.expected | 29 ---
12 files changed, 1 insertion(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 0eb1af5..79de357 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -66,7 +66,6 @@ public class ClassUtil {
classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep");
classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterMergeStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterMergeStep");
classRenameMap.put("org.apache.kylin.rest.util.KeywordDefaultDirtyHack", "org.apache.kylin.query.util.KeywordDefaultDirtyHack");
- classRenameMap.put("org.apache.kylin.rest.util.CognosParenthesesEscape", "org.apache.kylin.query.util.CognosParenthesesEscape");
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
deleted file mode 100644
index 8c6d82d..0000000
--- a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query.util;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Stack;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.collect.Lists;
-
-public class CognosParenthesesEscape implements QueryUtil.IQueryTransformer {
- private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\s*\\(\\s*)+(?!\\s*select\\s)", Pattern.CASE_INSENSITIVE);
-
- @Override
- public String transform(String sql, String project) {
- if (sql == null || sql.isEmpty()) {
- return sql;
- }
-
- Map<Integer, Integer> parenthesesPairs = findParenthesesPairs(sql);
- if (parenthesesPairs.isEmpty()) {
- // parentheses not found
- return sql;
- }
-
- List<Integer> parentheses = Lists.newArrayList();
- StringBuilder result = new StringBuilder(sql);
-
- Matcher m;
- while (true) {
- m = FROM_PATTERN.matcher(sql);
- if (!m.find()) {
- break;
- }
-
- int i = m.end() - 1;
- while (i > m.start()) {
- if (sql.charAt(i) == '(') {
- parentheses.add(i);
- }
- i--;
- }
-
- if (m.end() < sql.length()) {
- sql = sql.substring(m.end());
- } else {
- break;
- }
- }
-
- Collections.sort(parentheses);
- for (int i = 0; i < parentheses.size(); i++) {
- result.deleteCharAt(parentheses.get(i) - i);
- result.deleteCharAt(parenthesesPairs.get(parentheses.get(i)) - i - 1);
- }
- return result.toString();
- }
-
- private Map<Integer, Integer> findParenthesesPairs(String sql) {
- Map<Integer, Integer> result = new HashMap<>();
- if (sql.length() > 1) {
- Stack<Integer> lStack = new Stack<>();
- boolean inStrVal = false;
- for (int i = 0; i < sql.length(); i++) {
- switch (sql.charAt(i)) {
- case '(':
- if (!inStrVal) {
- lStack.push(i);
- }
- break;
- case ')':
- if (!inStrVal && !lStack.empty()) {
- result.put(lStack.pop(), i);
- }
- break;
- case '\'':
- inStrVal = !inStrVal;
- break;
- default:
- break;
- }
- }
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
deleted file mode 100644
index ad42b96..0000000
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query.util;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlDataTypeSpec;
-import org.apache.calcite.sql.SqlDynamicParam;
-import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlIntervalQualifier;
-import org.apache.calcite.sql.SqlLiteral;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.parser.SqlParseException;
-import org.apache.calcite.sql.util.SqlVisitor;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Functions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Ordering;
-
-public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
- private static final Logger logger = LoggerFactory.getLogger(ConvertToComputedColumn.class);
-
- @Override
- public String transform(String sql, String project) {
- if (project == null) {
- return sql;
- }
- ImmutableSortedMap<String, String> computedColumns = getSortedComputedColumnWithProject(project);
- String s = replaceComputedColumn(sql, computedColumns);
- if (!StringUtils.equals(sql, s)) {
- logger.debug("sql changed");
- }
- return s;
- }
-
- static String replaceComputedColumn(String inputSql, ImmutableSortedMap<String, String> computedColumn) {
- if (inputSql == null) {
- return "";
- }
-
- if (computedColumn == null || computedColumn.isEmpty()) {
- return inputSql;
- }
- String result = inputSql;
- List<Pair<String, String>> toBeReplacedExp = new ArrayList<>(); //{"alias":"expression"}, like {"t1":"t1.a+t1.b+t1.c"}
-
- for (String ccExp : computedColumn.keySet()) {
- List<SqlNode> matchedNodes = new ArrayList<>();
- try {
- matchedNodes = getMatchedNodes(inputSql, computedColumn.get(ccExp));
- } catch (SqlParseException e) {
- logger.error("Convert to computedColumn Fail,parse sql fail ", e);
- return inputSql;
- }
- for (SqlNode node : matchedNodes) {
- Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, inputSql);
- int start = startEndPos.getLeft();
- int end = startEndPos.getRight();
- //add table alias like t1.column,if exists alias
- String alias = getTableAlias(node);
- toBeReplacedExp.add(Pair.of(alias, inputSql.substring(start, end)));
- }
- logger.debug("Computed column: " + ccExp + "'s matched list:" + toBeReplacedExp);
- //replace user's input sql
- for (Pair<String, String> toBeReplaced : toBeReplacedExp) {
- result = result.replace(toBeReplaced.getRight(), toBeReplaced.getLeft() + ccExp);
- }
- }
- return result;
- }
-
- //Return matched node's position and its alias(if exists).If can not find matches, return an empty capacity list
- private static List<SqlNode> getMatchedNodes(String inputSql, String ccExp) throws SqlParseException {
- if (ccExp == null || ccExp.equals("")) {
- return new ArrayList<>();
- }
- ArrayList<SqlNode> toBeReplacedNodes = new ArrayList<>();
- SqlNode ccNode = CalciteParser.getExpNode(ccExp);
- List<SqlNode> inputNodes = getInputTreeNodes(inputSql);
-
- // find whether user input sql's tree node equals computed columns's define expression
- for (SqlNode inputNode : inputNodes) {
- if (CalciteParser.isNodeEqual(inputNode, ccNode)) {
- toBeReplacedNodes.add(inputNode);
- }
- }
- return toBeReplacedNodes;
- }
-
- private static List<SqlNode> getInputTreeNodes(String inputSql) throws SqlParseException {
- SqlTreeVisitor stv = new SqlTreeVisitor();
- CalciteParser.parse(inputSql).accept(stv);
- return stv.getSqlNodes();
- }
-
- private static String getTableAlias(SqlNode node) {
- if (node instanceof SqlCall) {
- SqlCall call = (SqlCall) node;
- return getTableAlias(call.getOperandList());
- }
- if (node instanceof SqlIdentifier) {
- StringBuilder alias = new StringBuilder("");
- ImmutableList<String> names = ((SqlIdentifier) node).names;
- if (names.size() >= 2) {
- for (int i = 0; i < names.size() - 1; i++) {
- alias.append(names.get(i)).append(".");
- }
- }
- return alias.toString();
- }
- if (node instanceof SqlNodeList) {
- return "";
- }
- if (node instanceof SqlLiteral) {
- return "";
- }
- return "";
- }
-
- private static String getTableAlias(List<SqlNode> operands) {
- for (SqlNode operand : operands) {
- return getTableAlias(operand);
- }
- return "";
- }
-
- private ImmutableSortedMap<String, String> getSortedComputedColumnWithProject(String project) {
-
- Map<String, String> computedColumns = new HashMap<>();
-
- MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<DataModelDesc> dataModelDescs = metadataManager.getModels(project);
- for (DataModelDesc dataModelDesc : dataModelDescs) {
- for (ComputedColumnDesc computedColumnDesc : dataModelDesc.getComputedColumnDescs()) {
- computedColumns.put(computedColumnDesc.getColumnName(), computedColumnDesc.getExpression());
- }
- }
-
- return getMapSortedByValue(computedColumns);
- }
-
- static ImmutableSortedMap<String, String> getMapSortedByValue(Map<String, String> computedColumns) {
- if (computedColumns == null || computedColumns.isEmpty()) {
- return null;
- }
-
- Ordering<String> ordering = Ordering.from(new Comparator<String>() {
- @Override
- public int compare(String o1, String o2) {
- return Integer.compare(o1.replaceAll("\\s*", "").length(), o2.replaceAll("\\s*", "").length());
- }
- }).reverse().nullsLast().onResultOf(Functions.forMap(computedColumns, null)).compound(Ordering.natural());
- return ImmutableSortedMap.copyOf(computedColumns, ordering);
- }
-
-}
-
-class SqlTreeVisitor implements SqlVisitor<SqlNode> {
- private List<SqlNode> sqlNodes;
-
- SqlTreeVisitor() {
- this.sqlNodes = new ArrayList<>();
- }
-
- List<SqlNode> getSqlNodes() {
- return sqlNodes;
- }
-
- @Override
- public SqlNode visit(SqlNodeList nodeList) {
- sqlNodes.add(nodeList);
- for (int i = 0; i < nodeList.size(); i++) {
- SqlNode node = nodeList.get(i);
- node.accept(this);
- }
- return null;
- }
-
- @Override
- public SqlNode visit(SqlLiteral literal) {
- sqlNodes.add(literal);
- return null;
- }
-
- @Override
- public SqlNode visit(SqlCall call) {
- sqlNodes.add(call);
- for (SqlNode operand : call.getOperandList()) {
- if (operand != null) {
- operand.accept(this);
- }
- }
- return null;
- }
-
- @Override
- public SqlNode visit(SqlIdentifier id) {
- sqlNodes.add(id);
- return null;
- }
-
- @Override
- public SqlNode visit(SqlDataTypeSpec type) {
- return null;
- }
-
- @Override
- public SqlNode visit(SqlDynamicParam param) {
- return null;
- }
-
- @Override
- public SqlNode visit(SqlIntervalQualifier intervalQualifier) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index eff9f6d..29e2c26 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -76,7 +76,7 @@ public class QueryUtil {
IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
transformers.add(t);
} catch (Exception e) {
- logger.error("Failed to init query transformer", e);
+ throw new RuntimeException("Failed to init query transformer", e);
}
}
queryTransformers = transformers;
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java b/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
deleted file mode 100644
index 7825dbd..0000000
--- a/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Collection;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class CognosParenthesesEscapeTest {
-
- @Test
- public void basicTest() {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String data = " from ((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
- String expected = " from a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
- String transformed = escape.transform(data, null);
- Assert.assertEquals(expected, transformed);
- }
-
- @Test
- public void advanced1Test() throws IOException {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"),
- Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"),
- Charset.defaultCharset());
- String transformed = escape.transform(query, null);
- //System.out.println(transformed);
- Assert.assertEquals(expected, transformed);
- }
-
- @Test
- public void advanced2Test() throws IOException {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query02.sql"),
- Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query02.sql.expected"),
- Charset.defaultCharset());
- String transformed = escape.transform(query, null);
- //System.out.println(transformed);
- Assert.assertEquals(expected, transformed);
- }
-
- @Test
- public void advanced3Test() throws IOException {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query03.sql"),
- Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query03.sql.expected"),
- Charset.defaultCharset());
- String transformed = escape.transform(query, null);
- //System.out.println(transformed);
- Assert.assertEquals(expected, transformed);
- }
-
- @Test
- public void proguardTest() throws IOException {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" },
- true);
- for (File f : files) {
- System.out.println("checking " + f.getAbsolutePath());
- String query = FileUtils.readFileToString(f, Charset.defaultCharset());
- String transformed = escape.transform(query, null);
- Assert.assertEquals(query, transformed);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
deleted file mode 100644
index d9910fe..0000000
--- a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query.util;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.calcite.sql.parser.SqlParseException;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableSortedMap;
-
-public class ConvertToComputedColumnTest {
-
- @Test
- public void testErrorCase() {
- //computed column is null or empty
- String sql = "select a from t";
- Map<String, String> map = new HashMap<>();
- ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
- Assert.assertEquals("select a from t", ConvertToComputedColumn.replaceComputedColumn(sql, null));
- Assert.assertEquals("select a from t", ConvertToComputedColumn.replaceComputedColumn(sql, computedColumns));
-
- //input is null or empty or parse error
- String sql1 = "";
- String sql2 = "select sum(a from t";
- Map<String, String> map2 = new HashMap<>();
- map2.put("cc", "a + b");
- ImmutableSortedMap<String, String> computedColumns2 = ConvertToComputedColumn.getMapSortedByValue(map2);
- Assert.assertEquals("", ConvertToComputedColumn.replaceComputedColumn(null, computedColumns2));
- Assert.assertEquals("", ConvertToComputedColumn.replaceComputedColumn(sql1, computedColumns2));
- Assert.assertEquals("select sum(a from t",
- ConvertToComputedColumn.replaceComputedColumn(sql2, computedColumns2));
- }
-
- @Test
- public void testReplaceComputedColumn() throws SqlParseException {
- String sql0 = "select (\"DB\".\"t1\" . \"a\" + DB.t1.b + DB.t1.c) as c, substring(substring(lstg_format_name,1,3),1,3) as d from table1 as t1 group by t1.a+ t1.b + t1.c having t1.a+t1.b+t1.c > 100 order by t1.a +t1.b +t1.c";
- String sql1 = "select sum(sum(a)) from t";
- String sql2 = "select t1.a + t1.b as aa, t2.c + t2.d as bb from table1 t1,table2 t2 where t1.a + t1.b > t2.c + t2.d order by t1.a + t1.b";
- String sql3 = "select substring(substring(lstg_format_name,1,3),1,3) from a";
-
- String expr0 = "a + b + c";
- String expr1 = "sum(a)";
- String expr2 = "a + b";
- String expr3 = "c + d";
- String expr = "substring(substring(lstg_format_name,1,3),1,3)";
-
- Map<String, String> map = new HashMap<>();
- map.put("cc0", expr0);
- map.put("cc1", expr1);
- map.put("cc2", expr2);
- map.put("cc3", expr3);
- map.put("cc", expr);
-
- ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
- Assert.assertEquals(
- "select (DB.t1.cc0) as c, cc as d from table1 as t1 group by T1.cc0 having T1.cc0 > 100 order by T1.cc0",
- ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns));
- Assert.assertEquals("select sum(cc1) from t",
- ConvertToComputedColumn.replaceComputedColumn(sql1, computedColumns));
- Assert.assertEquals(
- "select T1.cc2 as aa, T2.cc3 as bb from table1 t1,table2 t2 where T1.cc2 > T2.cc3 order by T1.cc2",
- ConvertToComputedColumn.replaceComputedColumn(sql2, computedColumns));
- Assert.assertEquals("select cc from a", ConvertToComputedColumn.replaceComputedColumn(sql3, computedColumns));
-
- }
-
- @Test
- public void testTwoCCHasSameSubExp() {
- String sql0 = "select a + b + c from t order by a + b";
-
- String expr0 = "a + b";
- String expr1 = "a + b + c";
-
- Map<String, String> map = new HashMap<>();
- map.put("cc1", expr0);
- map.put("cc0", expr1);
- ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
- Assert.assertEquals("select cc0 from t order by cc1",
- ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns));
-
- //防止添加的顺序造成影响
- String expr11 = "a + b + c";
- String expr00 = "a + b";
-
- Map<String, String> map2 = new HashMap<>();
- map2.put("cc0", expr11);
- map2.put("cc1", expr00);
- ImmutableSortedMap<String, String> computedColumns1 = ConvertToComputedColumn.getMapSortedByValue(map2);
- Assert.assertEquals("select cc0 from t order by cc1",
- ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns1));
-
- }
-
- @Test
- public void testCCWithBrackets() {
- String sql0 = "select ( a + b) + (c+d \t\n) from t";
- String expr1 = "a + b + (c + d)";
-
- Map<String, String> map = new HashMap<>();
- map.put("cc0", expr1);
- ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
- Assert.assertEquals("select cc0 from t",
- ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/resources/query/cognos/query01.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query01.sql b/query/src/test/resources/query/cognos/query01.sql
deleted file mode 100644
index a76b4e1..0000000
--- a/query/src/test/resources/query/cognos/query01.sql
+++ /dev/null
@@ -1,28 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "TABLE1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
- "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1")
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/resources/query/cognos/query01.sql.expected
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query01.sql.expected b/query/src/test/resources/query/cognos/query01.sql.expected
deleted file mode 100644
index c8005fc..0000000
--- a/query/src/test/resources/query/cognos/query01.sql.expected
+++ /dev/null
@@ -1,28 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "TABLE1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
- "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1"
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/resources/query/cognos/query02.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query02.sql b/query/src/test/resources/query/cognos/query02.sql
deleted file mode 100644
index 935251d..0000000
--- a/query/src/test/resources/query/cognos/query02.sql
+++ /dev/null
@@ -1,27 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "X1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN (SELECT "DIM1_1", "PK_1" FROM COGNOS"."TABLE1" WHERE DIM1_1 = '1') "X1" ON "FACT"."FK_1" = "X1"."PK_1")
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"X1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/resources/query/cognos/query02.sql.expected
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query02.sql.expected b/query/src/test/resources/query/cognos/query02.sql.expected
deleted file mode 100644
index 4d4f016..0000000
--- a/query/src/test/resources/query/cognos/query02.sql.expected
+++ /dev/null
@@ -1,27 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "X1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN (SELECT "DIM1_1", "PK_1" FROM COGNOS"."TABLE1" WHERE DIM1_1 = '1') "X1" ON "FACT"."FK_1" = "X1"."PK_1"
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"X1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/resources/query/cognos/query03.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query03.sql b/query/src/test/resources/query/cognos/query03.sql
deleted file mode 100644
index bfa483f..0000000
--- a/query/src/test/resources/query/cognos/query03.sql
+++ /dev/null
@@ -1,29 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "X1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM
- ( (
- SELECT "F1" FROM "T1") X0 LEFT OUTER JOIN (SELECT "DIM1_1", "PK_1" FROM COGNOS"."TABLE1" WHERE DIM1_1 = '1') "X1" ON "FACT"."FK_1" = "X1"."PK_1")
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"X1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf887a7a/query/src/test/resources/query/cognos/query03.sql.expected
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query03.sql.expected b/query/src/test/resources/query/cognos/query03.sql.expected
deleted file mode 100644
index 20215a9..0000000
--- a/query/src/test/resources/query/cognos/query03.sql.expected
+++ /dev/null
@@ -1,29 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "X1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM
- (
- SELECT "F1" FROM "T1") X0 LEFT OUTER JOIN (SELECT "DIM1_1", "PK_1" FROM COGNOS"."TABLE1" WHERE DIM1_1 = '1') "X1" ON "FACT"."FK_1" = "X1"."PK_1"
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"X1"."DIM1_1";