You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2017/09/08 01:53:45 UTC

[1/2] kylin git commit: APACHE-KYLIN-2725: add precision for percentile measures of system cubes

Repository: kylin
Updated Branches:
  refs/heads/yaho-cube-planner 0b1730ce4 -> 415593638


APACHE-KYLIN-2725: add precision for percentile measures of system cubes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f5fc510
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f5fc510
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f5fc510

Branch: refs/heads/yaho-cube-planner
Commit: 0f5fc5106c954e5a0f450dfbe92a8ef0a7a03868
Parents: 0b1730c
Author: Zhong <nj...@apache.org>
Authored: Thu Sep 7 17:36:58 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Thu Sep 7 17:39:47 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5fc510/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
index dad1629..e6914b4 100644
--- a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
+++ b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
@@ -626,7 +626,7 @@ public class CubeDescCreator {
         FunctionDesc function = new FunctionDesc();
         function.setExpression(PercentileMeasureType.FUNC_PERCENTILE);
         function.setParameter(parameterDesc);
-        function.setReturnType(PercentileMeasureType.DATATYPE_PERCENTILE);
+        function.setReturnType("percentile(100)");
 
         MeasureDesc result = new MeasureDesc();
         result.setName(column + "_PERCENTILE");


[2/2] kylin git commit: APACHE-KYLIN-2732: minor fix for DoggedCubeBuilder

Posted by nj...@apache.org.
APACHE-KYLIN-2732: minor fix for DoggedCubeBuilder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41559363
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41559363
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41559363

Branch: refs/heads/yaho-cube-planner
Commit: 41559363891cde3d73bd2a39eede037f85bc09f4
Parents: 0f5fc51
Author: Zhong <nj...@apache.org>
Authored: Thu Sep 7 17:36:45 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Thu Sep 7 17:39:59 2017 +0800

----------------------------------------------------------------------
 .../cube/inmemcubing/DoggedCubeBuilder.java     | 66 +++-----------------
 1 file changed, 10 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/41559363/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index b196753..ccd7137 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -31,7 +31,6 @@ import java.util.concurrent.ConcurrentNavigableMap;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequestBuilder;
@@ -88,32 +87,23 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
             logger.info("Dogged Cube Build start");
 
             try {
-                SplitThread last = null;
                 while (true) {
-                    if (last != null && shouldCutSplit(splits)) {
-                        cutSplit(last);
-                        last = null;
-                    }
+                    SplitThread last = new SplitThread(splits.size() + 1, RecordConsumeBlockingQueueController
+                            .getQueueController(inputConverterUnit, input, unitRows));
+                    splits.add(last);
+
+                    last.start();
+                    logger.info("Split #" + splits.size() + " kickoff");
+
+                    // Build splits sequentially
+                    last.join();
+
                     checkException(splits);
-                    if (last == null) {
-                        last = new SplitThread(splits.size() + 1, RecordConsumeBlockingQueueController
-                                .getQueueController(inputConverterUnit, input, unitRows));
-                        splits.add(last);
-                        last.start();
-                        logger.info("Split #" + splits.size() + " kickoff");
-                    }
                     if (last.inputController.ifEnd()) {
                         break;
                     }
-                    Thread.sleep(100);
                 }
 
-                logger.info("Has finished feeding data!!!");
-
-                for (SplitThread split : splits) {
-                    split.join();
-                }
-                checkException(splits);
                 logger.info("Dogged Cube Build splits complete, took " + (System.currentTimeMillis() - start) + " ms");
 
                 merger.mergeAndOutput(splits, output);
@@ -201,42 +191,6 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
                 throw new IOException(errors.size() + " exceptions during in-mem cube build, cause set to the first, check log for more", errors.get(0));
             }
         }
-
-        private void cutSplit(SplitThread last) {
-            try {
-                // signal the end of input
-                while (last.isAlive()) {
-                    last.inputController.forceCutPipe();
-                    break;
-                }
-
-                // wait cuboid build done
-                last.join();
-
-            } catch (InterruptedException e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        private boolean shouldCutSplit(List<SplitThread> splits) {
-            int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
-            int nSplit = splits.size();
-            long splitRowCount = nSplit == 0 ? 0 : splits.get(nSplit - 1).inputController.getOutputRowCountAfterCut();
-
-            logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold");
-
-            if (splitRowCount >= splitRowThreshold) {
-                logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold);
-                return true;
-            }
-
-            if (systemAvailMB <= reserveMemoryMB * 1.5) {
-                logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + "*1.5 MB");
-                return true;
-            }
-
-            return false;
-        }
     }
 
     private class SplitThread extends Thread {