You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/01 02:54:38 UTC
[02/50] [abbrv] kylin git commit: KYLIN-2783 merge
AggregationGroupScheduler into CuboidScheduler
KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8bab3df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8bab3df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8bab3df
Branch: refs/heads/master
Commit: c8bab3df33edd5dc146a02f94caccd0d810b68bd
Parents: 5008866
Author: Li Yang <li...@apache.org>
Authored: Sun Aug 13 16:34:22 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Mon Aug 14 09:35:09 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinVersion.java | 24 ++-
.../apache/kylin/common/KylinVersionTest.java | 19 +-
.../cube/cuboid/AggregationGroupScheduler.java | 126 ------------
.../kylin/cube/cuboid/CuboidScheduler.java | 4 +
.../cube/cuboid/DefaultCuboidScheduler.java | 191 ++++++++++++++-----
.../kylin/cube/model/AggregationGroup.java | 10 +-
.../org/apache/kylin/cube/model/SelectRule.java | 6 +-
7 files changed, 187 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 960bafe..c6913fc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -37,10 +37,11 @@ public class KylinVersion implements Comparable {
private static final String COMMIT_SHA1_v15 = "commit_SHA1";
private static final String COMMIT_SHA1_v13 = "commit.sha1";
- public int major;
- public int minor;
- public int revision;
- public boolean isSnapshot;
+ public final int major;
+ public final int minor;
+ public final int revision;
+ public final int internal;
+ public final boolean isSnapshot;
public KylinVersion(String version) {
@@ -58,12 +59,13 @@ public class KylinVersion implements Comparable {
major = Integer.parseInt(splits[0]);
minor = Integer.parseInt(splits[1]);
- revision = splits.length < 3 ? 0 : Integer.parseInt(splits[2]);
+ revision = splits.length > 2 ? Integer.parseInt(splits[2]) : 0;
+ internal = splits.length > 3 ? Integer.parseInt(splits[3]) : 0;
}
@Override
public String toString() {
- return "" + major + "." + minor + "." + revision;
+ return "" + major + "." + minor + "." + revision + "." + internal + (isSnapshot ? "-SNAPSHOT" : "");
}
@Override
@@ -83,13 +85,17 @@ public class KylinVersion implements Comparable {
if (comp != 0)
return comp;
+ comp = this.internal - v.internal;
+ if (comp != 0)
+ return comp;
+
return (this.isSnapshot ? 0 : 1) - (v.isSnapshot ? 0 : 1);
}
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
- private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0");
+ private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0.20403");
private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
@@ -125,6 +131,10 @@ public class KylinVersion implements Comparable {
public static boolean isBefore200(String ver) {
return new KylinVersion(ver).compareTo(VERSION_200) < 0;
}
+
+ public static int compare(String v1, String v2) {
+ return new KylinVersion(v1).compareTo(new KylinVersion(v2));
+ }
public static void main(String[] args) {
System.out.println(getKylinClientInformation());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
index 96b3189..6e903c8 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
@@ -37,4 +37,21 @@ public class KylinVersionTest {
Assert.assertEquals(1, ver1.minor);
Assert.assertEquals(0, ver1.revision);
}
-}
+
+ @Test
+ public void testToString() {
+ KylinVersion ver1 = new KylinVersion("2.1.7.321");
+ Assert.assertEquals(2, ver1.major);
+ Assert.assertEquals(1, ver1.minor);
+ Assert.assertEquals(7, ver1.revision);
+ Assert.assertEquals(321, ver1.internal);
+ Assert.assertEquals("2.1.7.321", ver1.toString());
+ }
+
+ @Test
+ public void testCompare() {
+ Assert.assertEquals(true, KylinVersion.isBefore200("1.9.9"));
+ Assert.assertEquals(false, KylinVersion.isBefore200("2.0.0"));
+ Assert.assertEquals(true, new KylinVersion("2.1.0").compareTo(new KylinVersion("2.1.0.123")) < 0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
deleted file mode 100644
index cdcbcfc..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.cuboid;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import org.apache.kylin.cube.model.AggregationGroup;
-import org.apache.kylin.cube.model.CubeDesc;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class AggregationGroupScheduler {
- /**
- * Get all valid cuboids for agg group, ignoring padding
- * @param agg agg group
- * @return cuboidId list
- */
- public static List<Long> getCuboidsForAgg(final CubeDesc cubeDesc, AggregationGroup agg) {
- Set<Long> cuboidHolder = new HashSet<>();
-
- // build tree structure
- Set<Long> children = getLowestCuboids(agg);
- while (!children.isEmpty()) {
- if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination()) {
- throw new IllegalStateException("too many combination for the aggregation group");
- }
- cuboidHolder.addAll(children);
- children = getOnTreeParentsByLayer(children, agg);
- }
-
- return Lists.newArrayList(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() {
- @Override
- public boolean apply(@Nullable Long cuboidId) {
- return !cubeDesc.isBlackedCuboid(cuboidId);
- }
- }));
- }
-
- private static Set<Long> getOnTreeParentsByLayer(Collection<Long> children, final AggregationGroup agg) {
- Set<Long> parents = new HashSet<>();
- for (long child : children) {
- parents.addAll(getOnTreeParents(child, agg));
- }
- parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>() {
- @Override
- public boolean apply(@Nullable Long cuboidId) {
- return agg.checkDimCap(cuboidId);
- }
- }));
- return parents;
- }
-
- private static Set<Long> getLowestCuboids(AggregationGroup agg) {
- return getOnTreeParents(0L, agg);
- }
-
- public static Set<Long> getOnTreeParents(long child, AggregationGroup agg) {
- Set<Long> parentCandidate = new HashSet<>();
-
- long tmpChild = child;
- if (tmpChild == agg.getPartialCubeFullMask()) {
- return parentCandidate;
- }
-
- if (agg.getMandatoryColumnMask() != 0L) {
- if (agg.isMandatoryOnlyValid()) {
- if (fillBit(tmpChild, agg.getMandatoryColumnMask(), parentCandidate)) {
- return parentCandidate;
- }
- } else {
- tmpChild |= agg.getMandatoryColumnMask();
- }
- }
-
- for (Long normal : agg.getNormalDims()) {
- fillBit(tmpChild, normal, parentCandidate);
- }
-
- for (Long joint : agg.getJoints()) {
- fillBit(tmpChild, joint, parentCandidate);
- }
-
- for (AggregationGroup.HierarchyMask hierarchy : agg.getHierarchyMasks()) {
- for (long mask : hierarchy.allMasks) {
- if (fillBit(tmpChild, mask, parentCandidate)) {
- break;
- }
- }
- }
-
- return parentCandidate;
- }
-
- private static boolean fillBit(long origin, long other, Set<Long> coll) {
- if ((origin & other) != other) {
- coll.add(origin | other);
- return true;
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 1d8f589..e802230 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Set;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
import com.google.common.base.Preconditions;
@@ -62,6 +63,9 @@ abstract public class CuboidScheduler {
/** Returns a cuboid on the tree that best matches the request cuboid. */
abstract public long findBestMatchCuboid(long requestCuboid);
+
+ /** (AggGroupScheduler) Calculate the cuboid set defined by an aggregation group. */
+ abstract public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg);
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index b75acd5..1c18b69 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -48,7 +48,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
private final Set<Long> allCuboidIds;
private final Map<Long, List<Long>> parent2child;
-
public DefaultCuboidScheduler(CubeDesc cubeDesc) {
super(cubeDesc);
@@ -84,6 +83,46 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
}
/**
+ * Get the parent cuboid really on the spanning tree.
+ * @param child an on-tree cuboid
+ * @return
+ */
+ @Override
+ public long findBestMatchCuboid(long child) {
+ long parent = getOnTreeParent(child);
+ while (parent > 0) {
+ if (cubeDesc.getAllCuboids().contains(parent)) {
+ break;
+ }
+ parent = getOnTreeParent(parent);
+ }
+
+ if (parent <= 0) {
+ throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
+ }
+ return parent;
+ }
+
+ private long getOnTreeParent(long child) {
+ Collection<Long> candidates = getOnTreeParents(child);
+ if (candidates == null || candidates.isEmpty()) {
+ return -1;
+ }
+ return Collections.min(candidates, Cuboid.cuboidSelectComparator);
+ }
+
+ private Set<Long> getOnTreeParents(long child) {
+ List<AggregationGroup> aggrs = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ if (agg.isOnTree(child)) {
+ aggrs.add(agg);
+ }
+ }
+
+ return getOnTreeParents(child, aggrs);
+ }
+
+ /**
* Collect cuboid from bottom up, considering all factor including black list
* Build tree steps:
* 1. Build tree from bottom up considering dim capping
@@ -94,7 +133,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
* and grandparent are missing, add grandparent to the tree.
* @return Cuboid collection
*/
- private Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp() {
+ protected Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp() {
int forward = cubeDesc.getParentForward();
KylinConfig config = cubeDesc.getConfig();
@@ -159,35 +198,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
}
/**
- * Get the parent cuboid really on the spanning tree.
- * @param child an on-tree cuboid
- * @return
- */
- @Override
- public long findBestMatchCuboid(long child) {
- long parent = getOnTreeParent(child);
- while (parent > 0) {
- if (cubeDesc.getAllCuboids().contains(parent)) {
- break;
- }
- parent = getOnTreeParent(parent);
- }
-
- if (parent <= 0) {
- throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
- }
- return parent;
- }
-
- private long getOnTreeParent(long child) {
- Collection<Long> candidates = getOnTreeParents(child);
- if (candidates == null || candidates.isEmpty()) {
- return -1;
- }
- return Collections.min(candidates, Cuboid.cuboidSelectComparator);
- }
-
- /**
* Get all parent for children cuboids, considering dim cap.
* @param children children cuboids
* @return all parents cuboids
@@ -205,7 +215,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
}
for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
- if (agg.isOnTree(cuboidId) && agg.checkDimCap(cuboidId)) {
+ if (agg.isOnTree(cuboidId) && checkDimCap(agg, cuboidId)) {
return true;
}
}
@@ -217,22 +227,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
}
/**
- * Get all *possible* parent for a child cuboid
- * @param child Child cuboid ID
- * @return all *possible* parent cuboids
- */
- private Set<Long> getOnTreeParents(long child) {
- List<AggregationGroup> aggrs = Lists.newArrayList();
- for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
- if (agg.isOnTree(child)) {
- aggrs.add(agg);
- }
- }
-
- return getOnTreeParents(child, aggrs);
- }
-
- /**
* Get lowest (not Cube building level) Cuboids for every Agg group
* @return lowest level cuboids
*/
@@ -252,9 +246,108 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
parentCandidate.add(Cuboid.getBaseCuboidId(cubeDesc));
return parentCandidate;
}
- parentCandidate.addAll(AggregationGroupScheduler.getOnTreeParents(child, agg));
+ parentCandidate.addAll(getOnTreeParents(child, agg));
+ }
+
+ return parentCandidate;
+ }
+
+ /**
+ * Get all valid cuboids for agg group, ignoring padding
+ * @param agg agg group
+ * @return cuboidId list
+ */
+ @Override
+ public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) {
+ Set<Long> cuboidHolder = new HashSet<>();
+
+ // build tree structure
+ Set<Long> children = getLowestCuboids(agg);
+ while (!children.isEmpty()) {
+ if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination()) {
+ throw new IllegalStateException("too many combination for the aggregation group");
+ }
+ cuboidHolder.addAll(children);
+ children = getOnTreeParentsByLayer(children, agg);
+ }
+
+ return Sets.newHashSet(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() {
+ @Override
+ public boolean apply(@Nullable Long cuboidId) {
+ return !cubeDesc.isBlackedCuboid(cuboidId);
+ }
+ }));
+ }
+
+ private Set<Long> getOnTreeParentsByLayer(Collection<Long> children, final AggregationGroup agg) {
+ Set<Long> parents = new HashSet<>();
+ for (long child : children) {
+ parents.addAll(getOnTreeParents(child, agg));
+ }
+ parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>() {
+ @Override
+ public boolean apply(@Nullable Long cuboidId) {
+ return checkDimCap(agg, cuboidId);
+ }
+ }));
+ return parents;
+ }
+
+ private Set<Long> getLowestCuboids(AggregationGroup agg) {
+ return getOnTreeParents(0L, agg);
+ }
+
+ private Set<Long> getOnTreeParents(long child, AggregationGroup agg) {
+ Set<Long> parentCandidate = new HashSet<>();
+
+ long tmpChild = child;
+ if (tmpChild == agg.getPartialCubeFullMask()) {
+ return parentCandidate;
+ }
+
+ if (agg.getMandatoryColumnMask() != 0L) {
+ if (agg.isMandatoryOnlyValid()) {
+ if (fillBit(tmpChild, agg.getMandatoryColumnMask(), parentCandidate)) {
+ return parentCandidate;
+ }
+ } else {
+ tmpChild |= agg.getMandatoryColumnMask();
+ }
+ }
+
+ for (Long normal : agg.getNormalDims()) {
+ fillBit(tmpChild, normal, parentCandidate);
+ }
+
+ for (Long joint : agg.getJoints()) {
+ fillBit(tmpChild, joint, parentCandidate);
+ }
+
+ for (AggregationGroup.HierarchyMask hierarchy : agg.getHierarchyMasks()) {
+ for (long mask : hierarchy.allMasks) {
+ if (fillBit(tmpChild, mask, parentCandidate)) {
+ break;
+ }
+ }
}
return parentCandidate;
}
+
+ private boolean fillBit(long origin, long other, Set<Long> coll) {
+ if ((origin & other) != other) {
+ coll.add(origin | other);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkDimCap(AggregationGroup agg, long cuboidID) {
+ int dimCap = agg.getDimCap();
+ if (dimCap <= 0)
+ return true;
+
+ return Long.bitCount(cuboidID) <= dimCap;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 0533ea1..af026af 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.kylin.cube.cuboid.AggregationGroupScheduler;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.TblColRef;
@@ -296,7 +295,7 @@ public class AggregationGroup implements Serializable {
long combination = 1;
if (this.getDimCap() > 0) {
- combination = AggregationGroupScheduler.getCuboidsForAgg(cubeDesc, this).size();
+ combination = cubeDesc.getCuboidScheduler().calculateCuboidsForAggGroup(this).size();
} else {
Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
@@ -389,13 +388,6 @@ public class AggregationGroup implements Serializable {
return true;
}
- public boolean checkDimCap(long cuboidID) {
- if (getDimCap() <= 0) {
- return true;
- }
- return Long.bitCount(cuboidID) <= getDimCap();
- }
-
public void setIncludes(String[] includes) {
this.includes = includes;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
index 80052af..d78da9f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
@@ -18,10 +18,14 @@
package org.apache.kylin.cube.model;
+import java.io.Serializable;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-public class SelectRule implements java.io.Serializable {
+public class SelectRule implements Serializable {
+ private static final long serialVersionUID = 1L;
+
@JsonProperty("hierarchy_dims")
public String[][] hierarchyDims;
@JsonProperty("mandatory_dims")