You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2017/09/08 01:53:45 UTC
[1/2] kylin git commit: APACHE-KYLIN-2725: add precision for
percentile measures of system cubes
Repository: kylin
Updated Branches:
refs/heads/yaho-cube-planner 0b1730ce4 -> 415593638
APACHE-KYLIN-2725: add precision for percentile measures of system cubes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f5fc510
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f5fc510
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f5fc510
Branch: refs/heads/yaho-cube-planner
Commit: 0f5fc5106c954e5a0f450dfbe92a8ef0a7a03868
Parents: 0b1730c
Author: Zhong <nj...@apache.org>
Authored: Thu Sep 7 17:36:58 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Thu Sep 7 17:39:47 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5fc510/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
index dad1629..e6914b4 100644
--- a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
+++ b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
@@ -626,7 +626,7 @@ public class CubeDescCreator {
FunctionDesc function = new FunctionDesc();
function.setExpression(PercentileMeasureType.FUNC_PERCENTILE);
function.setParameter(parameterDesc);
- function.setReturnType(PercentileMeasureType.DATATYPE_PERCENTILE);
+ function.setReturnType("percentile(100)");
MeasureDesc result = new MeasureDesc();
result.setName(column + "_PERCENTILE");
[2/2] kylin git commit: APACHE-KYLIN-2732: minor fix for
DoggedCubeBuilder
Posted by nj...@apache.org.
APACHE-KYLIN-2732: minor fix for DoggedCubeBuilder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41559363
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41559363
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41559363
Branch: refs/heads/yaho-cube-planner
Commit: 41559363891cde3d73bd2a39eede037f85bc09f4
Parents: 0f5fc51
Author: Zhong <nj...@apache.org>
Authored: Thu Sep 7 17:36:45 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Thu Sep 7 17:39:59 2017 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/DoggedCubeBuilder.java | 66 +++-----------------
1 file changed, 10 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/41559363/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index b196753..ccd7137 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -31,7 +31,6 @@ import java.util.concurrent.ConcurrentNavigableMap;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
@@ -88,32 +87,23 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
logger.info("Dogged Cube Build start");
try {
- SplitThread last = null;
while (true) {
- if (last != null && shouldCutSplit(splits)) {
- cutSplit(last);
- last = null;
- }
+ SplitThread last = new SplitThread(splits.size() + 1, RecordConsumeBlockingQueueController
+ .getQueueController(inputConverterUnit, input, unitRows));
+ splits.add(last);
+
+ last.start();
+ logger.info("Split #" + splits.size() + " kickoff");
+
+ // Build splits sequentially
+ last.join();
+
checkException(splits);
- if (last == null) {
- last = new SplitThread(splits.size() + 1, RecordConsumeBlockingQueueController
- .getQueueController(inputConverterUnit, input, unitRows));
- splits.add(last);
- last.start();
- logger.info("Split #" + splits.size() + " kickoff");
- }
if (last.inputController.ifEnd()) {
break;
}
- Thread.sleep(100);
}
- logger.info("Has finished feeding data!!!");
-
- for (SplitThread split : splits) {
- split.join();
- }
- checkException(splits);
logger.info("Dogged Cube Build splits complete, took " + (System.currentTimeMillis() - start) + " ms");
merger.mergeAndOutput(splits, output);
@@ -201,42 +191,6 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
throw new IOException(errors.size() + " exceptions during in-mem cube build, cause set to the first, check log for more", errors.get(0));
}
}
-
- private void cutSplit(SplitThread last) {
- try {
- // signal the end of input
- while (last.isAlive()) {
- last.inputController.forceCutPipe();
- break;
- }
-
- // wait cuboid build done
- last.join();
-
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- private boolean shouldCutSplit(List<SplitThread> splits) {
- int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
- int nSplit = splits.size();
- long splitRowCount = nSplit == 0 ? 0 : splits.get(nSplit - 1).inputController.getOutputRowCountAfterCut();
-
- logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold");
-
- if (splitRowCount >= splitRowThreshold) {
- logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold);
- return true;
- }
-
- if (systemAvailMB <= reserveMemoryMB * 1.5) {
- logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + "*1.5 MB");
- return true;
- }
-
- return false;
- }
}
private class SplitThread extends Thread {