You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2017/09/08 03:34:57 UTC

[01/22] kylin git commit: APACHE-KYLIN-2730: Introduce genetic algorithm for cube planner [Forced Update!]

Repository: kylin
Updated Branches:
  refs/heads/yaho-cube-planner 415593638 -> bd3e7bf0c (forced update)


APACHE-KYLIN-2730: Introduce genetic algorithm for cube planner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3c4c56e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3c4c56e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3c4c56e

Branch: refs/heads/yaho-cube-planner
Commit: b3c4c56e5924a9a2386a6f9bb5cd43bffeb39c32
Parents: 74772d7
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 14:59:02 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:38 2017 +0800

----------------------------------------------------------------------
 .../algorithm/generic/BitsChromosome.java       | 133 +++++++++
 .../generic/CombinedStoppingCondition.java      |  43 +++
 .../cuboid/algorithm/generic/CuboidEncoder.java |  44 +++
 .../algorithm/generic/GeneticAlgorithm.java     | 283 +++++++++++++++++++
 .../generic/RouletteWheelSelection.java         |  61 ++++
 .../algorithm/generic/lib/BitsMutation.java     |  59 ++++
 .../algorithm/generic/lib/Chromosome.java       | 104 +++++++
 .../lib/ChromosomeMismatchException.java        |  48 ++++
 .../algorithm/generic/lib/ChromosomePair.java   |  67 +++++
 .../algorithm/generic/lib/CrossoverPolicy.java  |  37 +++
 .../generic/lib/ElitisticListPopulation.java    | 117 ++++++++
 .../cuboid/algorithm/generic/lib/Fitness.java   |  33 +++
 .../generic/lib/FixedGenerationCount.java       |  71 +++++
 .../algorithm/generic/lib/ListPopulation.java   | 223 +++++++++++++++
 .../algorithm/generic/lib/MutationPolicy.java   |  34 +++
 .../generic/lib/OnePointCrossover.java          | 125 ++++++++
 .../algorithm/generic/lib/Population.java       |  59 ++++
 .../algorithm/generic/lib/SelectionPolicy.java  |  33 +++
 .../generic/lib/StoppingCondition.java          |  34 +++
 .../generic/lib/TournamentSelection.java        | 114 ++++++++
 .../cuboid/algorithm/GeneticAlgorithmTest.java  |  57 ++++
 21 files changed, 1779 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java
new file mode 100755
index 0000000..6445ca6
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.BitSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidBenefitModel;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+
+import com.google.common.collect.Sets;
+
+public class BitsChromosome extends Chromosome {
+
+    private BitSet key;
+    private int length;
+    private BenefitPolicy benefitPolicy;
+    private CuboidStats cuboidStats;
+    private CuboidEncoder cuboidEncoder;
+    private double spaceLimit;
+    private double spaceCost;
+    private double fitness = -1;
+
+    public BitsChromosome(final BitSet key, final BenefitPolicy benefitPolicy, final CuboidStats cuboidStats,
+            final double spaceLimit) {
+        super();
+        this.key = key;
+        this.length = cuboidStats.getAllCuboidsForSelection().size();
+        this.benefitPolicy = benefitPolicy.getInstance();
+        this.cuboidStats = cuboidStats;
+        this.cuboidEncoder = new CuboidEncoder(cuboidStats);
+        this.spaceLimit = spaceLimit;
+        initSpaceCost();
+    }
+
+    public BitsChromosome newBitsChromosome(BitSet newkey) {
+        return new BitsChromosome(newkey, this.benefitPolicy, this.cuboidStats, this.spaceLimit);
+    }
+
+    private void initSpaceCost() {
+        spaceCost = 0;
+        List<Long> remainingCuboids = cuboidEncoder.toCuboidList(key);
+        for (Long cuboid : remainingCuboids) {
+            spaceCost += cuboidStats.getCuboidSize(cuboid);
+        }
+    }
+
+    public BitSet getKey() {
+        return key;
+    }
+
+    public int getLength() {
+        return length;
+    }
+
+    public CuboidEncoder getCuboidEncoder() {
+        return cuboidEncoder;
+    }
+
+    @Override
+    public double fitness() {
+        if (fitness == -1) {
+            fitness = calculateFitness();
+        }
+        return fitness;
+    }
+
+    @Override
+    protected boolean isSame(final Chromosome another) {
+        return this.equals(another);
+    }
+
+    private synchronized double calculateFitness() {
+        List<Long> remainingCuboids = cuboidEncoder.toCuboidList(key);
+        Set<Long> selectedCuboidSets = Sets.newHashSet();
+        selectedCuboidSets.addAll(cuboidStats.getAllCuboidsForMandatory());
+
+        CuboidBenefitModel.BenefitModel benefitModel = benefitPolicy.calculateBenefitTotal(remainingCuboids, selectedCuboidSets);
+        double totalBenefit = benefitModel.getBenefit();
+        if (spaceCost > spaceLimit) {
+            totalBenefit = totalBenefit * spaceLimit / spaceCost;
+        }
+        return totalBenefit;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((key == null) ? 0 : key.hashCode());
+        result = prime * result + length;
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        BitsChromosome other = (BitsChromosome) obj;
+        if (length != other.length) {
+            return false;
+        }
+        if (key == null) {
+            if (other.key != null)
+                return false;
+        } else if (!key.equals(other.key))
+            return false;
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
new file mode 100755
index 0000000..bc4bb99
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.StoppingCondition;
+
+import java.util.List;
+
+public class CombinedStoppingCondition implements StoppingCondition {
+    private List<StoppingCondition> conditions;
+
+    public CombinedStoppingCondition(List<StoppingCondition> conditions) {
+        this.conditions = conditions;
+    }
+
+    @Override
+    public boolean isSatisfied(Population population) {
+        for (StoppingCondition condition : conditions) {
+            if (condition.isSatisfied(population)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java
new file mode 100755
index 0000000..7e48413
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+
+import com.google.common.collect.Lists;
+
+public class CuboidEncoder {
+    private List<Long> selectionCuboids;
+
+    public CuboidEncoder(CuboidStats cuboidStats) {
+        selectionCuboids = Lists.newArrayList(cuboidStats.getAllCuboidsForSelection());
+        Collections.sort(selectionCuboids, Collections.reverseOrder());
+    }
+
+    public List<Long> toCuboidList(BitSet bits) {
+        List<Long> cuboids = Lists.newArrayListWithCapacity(bits.cardinality());
+        for (int i = bits.nextSetBit(0); i >= 0; i = bits.nextSetBit(i + 1)) {
+            cuboids.add(selectionCuboids.get(i));
+        }
+        return cuboids;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java
new file mode 100755
index 0000000..e2545b9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.BitSet;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.kylin.cube.cuboid.algorithm.AbstractRecommendAlgorithm;
+import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.BitsMutation;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ChromosomePair;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.CrossoverPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ElitisticListPopulation;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.FixedGenerationCount;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.MutationPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.OnePointCrossover;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.SelectionPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.StoppingCondition;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Implementation of a genetic algorithm to recommend a list of cuboids. All factors that govern the processing
+ * of the algorithm can be configured.
+ *
+ */
+public class GeneticAlgorithm extends AbstractRecommendAlgorithm {
+
+    private static final Logger logger = LoggerFactory.getLogger(GeneticAlgorithm.class);
+    public static ThreadLocal<Random> RANDGEN = new ThreadLocal<Random>() {
+        @Override
+        protected Random initialValue() {
+            return new Random(System.currentTimeMillis() * Thread.currentThread().getId());
+        }
+    };
+    /** the rate of crossover for the algorithm. */
+    private final double crossoverRate = 0.9;
+    /** the rate of mutation for the algorithm. */
+    private final double mutationRate = 0.001;
+    /** the init population size. */
+    private final int populationSize = 500;
+    /** the max population size. */
+    private final int maxPopulationSize = 510;
+    private CrossoverPolicy crossoverPolicy;
+    private MutationPolicy mutationPolicy;
+    private SelectionPolicy selectionPolicy;
+    private CuboidEncoder cuboidEncoder;
+    /** the number of generations evolved to reach {@link StoppingCondition} in the last run. */
+    private int generationsEvolved = 0;
+
+    public GeneticAlgorithm(final long timeout, BenefitPolicy benefitPolicy, CuboidStats cuboidStats) {
+        super(timeout, benefitPolicy, cuboidStats);
+        this.crossoverPolicy = new OnePointCrossover();
+        this.mutationPolicy = new BitsMutation();
+        this.selectionPolicy = new RouletteWheelSelection();
+
+        this.cuboidEncoder = new CuboidEncoder(getCuboidStats());
+    }
+
+    @Override
+    public List<Long> recommend(double expansionRate) {
+        double spaceLimit = getCuboidStats().getBaseCuboidSize() * expansionRate;
+        return start(spaceLimit);
+    }
+
+    @Override
+    public List<Long> start(double maxSpaceLimit) {
+        logger.info("Genetic Algorithm started.");
+
+        getBenefitPolicy().initBeforeStart();
+
+        //Initial mandatory cuboids
+        double remainingSpace = maxSpaceLimit;
+        for (Long mandatoryOne : getCuboidStats().getAllCuboidsForMandatory()) {
+            if (getCuboidStats().getCuboidSize(mandatoryOne) != null) {
+                remainingSpace -= getCuboidStats().getCuboidSize(mandatoryOne);
+            }
+        }
+
+        //Generate a population randomly
+        Population initial = initRandomPopulation(remainingSpace);
+
+        //Set stopping condition
+        List<StoppingCondition> conditions = Lists.newArrayList();
+        conditions.add(new FixedGenerationCount(550));
+        CombinedStoppingCondition stopCondition = new CombinedStoppingCondition(conditions);
+
+        //Start the evolution
+        Population current = evolve(initial, stopCondition);
+        BitsChromosome chromosome = (BitsChromosome) current.getFittestChromosome();
+        logger.info("Genetic Algorithm finished.");
+        List<Long> finalList = Lists.newArrayList();
+        finalList.addAll(getCuboidStats().getAllCuboidsForMandatory());
+        finalList.addAll(cuboidEncoder.toCuboidList(chromosome.getKey()));
+
+        double totalSpace = 0;
+        if (logger.isInfoEnabled()) {
+            for (Long cuboid : finalList) {
+                Double unitSpace = getCuboidStats().getCuboidSize(cuboid);
+                if (unitSpace != null) {
+                    logger.info(String.format("cuboidId %d and Space: %f", cuboid, unitSpace));
+                    totalSpace += unitSpace;
+                } else {
+                    logger.info(String.format("mandatory cuboidId %d", cuboid));
+                }
+            }
+            logger.info("Total Space:" + totalSpace);
+            logger.info("Space Expansion Rate:" + totalSpace / getCuboidStats().getBaseCuboidSize());
+        }
+        return finalList;
+    }
+
+    protected Population initRandomPopulation(double maxSpaceLimit) {
+        List<Chromosome> chromosomeList = Lists.newArrayListWithCapacity(populationSize);
+        List<Long> cuboidsForSelection = Lists.newArrayList(getCuboidStats().getAllCuboidsForSelection());
+        int selectionSize = cuboidsForSelection.size();
+
+        while (chromosomeList.size() < populationSize) {
+            BitSet bitSetForSelection = new BitSet(selectionSize);
+
+            //Initialize selection genes
+            double totalSpace = 0;
+            while (totalSpace < maxSpaceLimit) {
+                int j = RANDGEN.get().nextInt(selectionSize);
+                if (!bitSetForSelection.get(j)) {
+                    totalSpace += getCuboidStats().getCuboidSize(cuboidsForSelection.get(j));
+                    bitSetForSelection.set(j);
+                }
+            }
+
+            Chromosome chromosome = new BitsChromosome(bitSetForSelection, getBenefitPolicy(), getCuboidStats(),
+                    maxSpaceLimit);
+            chromosomeList.add(chromosome);
+        }
+        return new ElitisticListPopulation(chromosomeList, maxPopulationSize, 0.8);
+    }
+
+    /**
+     * Evolve the given population. Evolution stops when the stopping condition
+     * is satisfied. Updates the {@link #getGenerationsEvolved() generationsEvolved}
+     * property with the number of generations evolved before the StoppingCondition
+     * is satisfied.
+     *
+     * @param initial the initial, seed population.
+     * @param condition the stopping condition used to stop evolution.
+     * @return the population that satisfies the stopping condition.
+     */
+    protected Population evolve(final Population initial, final StoppingCondition condition) {
+        Population current = initial;
+        generationsEvolved = 0;
+        while (!condition.isSatisfied(current) && (!shouldCancel())) {
+            current = nextGeneration(current);
+            generationsEvolved++;
+            logger.info("Generations evolved count:" + generationsEvolved);
+        }
+        return current;
+    }
+
+    /**
+     * Evolve the given population into the next generation.
+     * <p>
+     * <ol>
+     *  <li>Get nextGeneration population to fill from <code>current</code>
+     *      generation, using its nextGeneration method</li>
+     *  <li>Loop until new generation is filled:</li>
+     *  <ul><li>Apply configured SelectionPolicy to select a pair of parents
+     *          from <code>current</code></li>
+     *      <li>With probability = {@link #getCrossoverRate()}, apply
+     *          configured {@link CrossoverPolicy} to parents</li>
+     *      <li>With probability = {@link #getMutationRate()}, apply
+     *          configured {@link MutationPolicy} to each of the offspring</li>
+     *      <li>Add offspring individually to nextGeneration,
+     *          space permitting</li>
+     *  </ul>
+     *  <li>Return nextGeneration</li>
+     * </ol>
+     *
+     * @param current the current population.
+     * @return the population for the next generation.
+     */
+    protected Population nextGeneration(final Population current) {
+        Population nextGeneration = current.nextGeneration();
+
+        while (nextGeneration.getPopulationSize() < nextGeneration.getPopulationLimit()) {
+            // select parent chromosomes
+            ChromosomePair pair = getSelectionPolicy().select(current);
+
+            // crossover?
+            if (RANDGEN.get().nextDouble() < getCrossoverRate()) {
+                // apply crossover policy to create two offspring
+                pair = getCrossoverPolicy().crossover(pair.getFirst(), pair.getSecond());
+            }
+
+            // mutation?
+            if (RANDGEN.get().nextDouble() < getMutationRate()) {
+                // apply mutation policy to the chromosomes
+                pair = new ChromosomePair(getMutationPolicy().mutate(pair.getFirst()),
+                        getMutationPolicy().mutate(pair.getSecond()));
+            }
+
+            // add the first chromosome to the population
+            nextGeneration.addChromosome(pair.getFirst());
+            // is there still a place for the second chromosome?
+            if (nextGeneration.getPopulationSize() < nextGeneration.getPopulationLimit()) {
+                // add the second chromosome to the population
+                nextGeneration.addChromosome(pair.getSecond());
+            }
+        }
+        return nextGeneration;
+    }
+
+    /**
+     * Returns the crossover policy.
+     * @return crossover policy
+     */
+    public CrossoverPolicy getCrossoverPolicy() {
+        return crossoverPolicy;
+    }
+
+    /**
+     * Returns the crossover rate.
+     * @return crossover rate
+     */
+    public double getCrossoverRate() {
+        return crossoverRate;
+    }
+
+    /**
+     * Returns the mutation policy.
+     * @return mutation policy
+     */
+    public MutationPolicy getMutationPolicy() {
+        return mutationPolicy;
+    }
+
+    /**
+     * Returns the mutation rate.
+     * @return mutation rate
+     */
+    public double getMutationRate() {
+        return mutationRate;
+    }
+
+    /**
+     * Returns the selection policy.
+     * @return selection policy
+     */
+    public SelectionPolicy getSelectionPolicy() {
+        return selectionPolicy;
+    }
+
+    /**
+     * Returns the number of generations evolved to reach {@link StoppingCondition} in the last run.
+     *
+     * @return number of generations evolved
+     */
+    public int getGenerationsEvolved() {
+        return generationsEvolved;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java
new file mode 100755
index 0000000..db84171
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ChromosomePair;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ListPopulation;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.SelectionPolicy;
+
+import com.google.common.collect.Lists;
+
+public class RouletteWheelSelection implements SelectionPolicy {
+
+    @Override
+    public ChromosomePair select(Population population) throws IllegalArgumentException {
+        // create a copy of the chromosome list
+        List<Chromosome> chromosomes = Lists.newArrayList(((ListPopulation) population).getChromosomes());
+
+        double maxFitness = 0;
+        double totalFitness = 0;
+        for (Chromosome o : chromosomes) {
+            double fitness = o.getFitness();
+            totalFitness += fitness;
+            if (fitness > maxFitness) {
+                maxFitness = fitness;
+            }
+        }
+        return new ChromosomePair(rouletteWheel(chromosomes, totalFitness), rouletteWheel(chromosomes, totalFitness));
+    }
+
+    private Chromosome rouletteWheel(final List<Chromosome> chromosomes, final double totalFitness) {
+        float rnd = (float) (GeneticAlgorithm.RANDGEN.get().nextDouble() * totalFitness);
+        float runningScore = 0;
+        for (Chromosome o : chromosomes) {
+            if (rnd >= runningScore && rnd <= runningScore + o.getFitness()) {
+                return o;
+            }
+            runningScore += o.getFitness();
+        }
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java
new file mode 100755
index 0000000..d9d13c1
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.BitSet;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.BitsChromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+
+/**
+ * Mutation for {@link BitsChromosome}s. Randomly changes one gene.
+ *
+ */
+public class BitsMutation implements MutationPolicy {
+
+    /**
+     * Mutate the given chromosome. Randomly changes one gene.
+     *
+     * @param original the original chromosome.
+     * @return the mutated chromosome.
+     * @throws IllegalArgumentException if <code>original</code> is not an instance of {@link BitsChromosome}.
+     */
+    public Chromosome mutate(Chromosome original) throws IllegalArgumentException {
+        if (!(original instanceof BitsChromosome)) {
+            throw new IllegalArgumentException("Chromosome must be of type BitsChromosome.");
+        }
+
+        BitsChromosome origChrom = (BitsChromosome) original;
+        BitSet newNey = (BitSet) origChrom.getKey().clone();
+
+        // randomly select a gene
+        int geneIndex = getMutationGeneIndex(origChrom);
+        // change it
+        boolean value = newNey.get(geneIndex);
+        newNey.set(geneIndex, !value);
+        Chromosome newChrom = origChrom.newBitsChromosome(newNey);
+        return newChrom;
+    }
+
+    protected int getMutationGeneIndex(BitsChromosome origChrom) {
+        return GeneticAlgorithm.RANDGEN.get().nextInt(origChrom.getLength());
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java
new file mode 100755
index 0000000..f8fa489
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Individual in a population. Chromosomes are compared based on their fitness.
+ * <p>
+ * The chromosomes are IMMUTABLE, and so their fitness is also immutable and
+ * therefore it can be cached.
+ *
+ */
+public abstract class Chromosome implements Comparable<Chromosome>, Fitness {
+    /** Value assigned when no fitness has been computed yet. */
+    private static final double NO_FITNESS = Double.NEGATIVE_INFINITY;
+
+    /** Cached value of the fitness of this chromosome. */
+    private double fitness = NO_FITNESS;
+
+    /**
+     * Access the fitness of this chromosome. The bigger the fitness, the better the chromosome.
+     * <p>
+     * Computation of fitness is usually very time-consuming task, therefore the fitness is cached.
+     *
+     * @return the fitness
+     */
+    public double getFitness() {
+        if (this.fitness == NO_FITNESS) {
+            // no cache - compute the fitness
+            this.fitness = fitness();
+        }
+        return this.fitness;
+    }
+
+    /**
+     * Compares two chromosomes based on their fitness. The bigger the fitness, the better the chromosome.
+     *
+     * @param another another chromosome to compare
+     * @return
+     * <ul>
+     *   <li>-1 if <code>another</code> is better than <code>this</code></li>
+     *   <li>1 if <code>another</code> is worse than <code>this</code></li>
+     *   <li>0 if the two chromosomes have the same fitness</li>
+     * </ul>
+     */
+    public int compareTo(final Chromosome another) {
+        return Double.compare(getFitness(), another.getFitness());
+    }
+
+    /**
+     * Returns <code>true</code> iff <code>another</code> has the same representation and therefore the same fitness. By
+     * default, it returns false -- override it in your implementation if you need it.
+     *
+     * @param another chromosome to compare
+     * @return true if <code>another</code> is equivalent to this chromosome
+     */
+    protected boolean isSame(final Chromosome another) {
+        return false;
+    }
+
+    /**
+     * Searches the <code>population</code> for another chromosome with the same representation. If such chromosome is
+     * found, it is returned, if no such chromosome exists, returns <code>null</code>.
+     *
+     * @param population Population to search
+     * @return Chromosome with the same representation, or <code>null</code> if no such chromosome exists.
+     */
+    protected Chromosome findSameChromosome(final Population population) {
+        for (Chromosome anotherChr : population) {
+            if (this.isSame(anotherChr)) {
+                return anotherChr;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Searches the population for a chromosome representing the same solution, and if it finds one,
+     * updates the fitness to its value.
+     *
+     * @param population Population to search
+     */
+    public void searchForFitnessUpdate(final Population population) {
+        Chromosome sameChromosome = findSameChromosome(population);
+        if (sameChromosome != null) {
+            fitness = sameChromosome.getFitness();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java
new file mode 100755
index 0000000..51d5cb3
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Exception to be thrown when two Chromosomes length differ.
+ *
+ */
+public class ChromosomeMismatchException extends IllegalArgumentException {
+    private static final long serialVersionUID = -7483865132286153255L;
+
+    private final int length;
+
+    /**
+     * Construct an exception from the mismatched chromosomes.
+     *
+     * @param errorMsg error info.
+     * @param wrong Wrong length.
+     * @param expected Expected length.
+     */
+    public ChromosomeMismatchException(String errorMsg, int wrong, int expected) {
+        super(errorMsg);
+        length = expected;
+    }
+
+    /**
+     * @return the expected length.
+     */
+    public int getLength() {
+        return length;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java
new file mode 100755
index 0000000..05da4b5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * A pair of {@link Chromosome} objects.
+ *
+ */
+public class ChromosomePair {
+    /** the first chromosome in the pair. */
+    private final Chromosome first;
+
+    /** the second chromosome in the pair. */
+    private final Chromosome second;
+
+    /**
+     * Create a chromosome pair.
+     *
+     * @param c1 the first chromosome.
+     * @param c2 the second chromosome.
+     */
+    public ChromosomePair(final Chromosome c1, final Chromosome c2) {
+        super();
+        first = c1;
+        second = c2;
+    }
+
+    /**
+     * Access the first chromosome.
+     *
+     * @return the first chromosome.
+     */
+    public Chromosome getFirst() {
+        return first;
+    }
+
+    /**
+     * Access the second chromosome.
+     *
+     * @return the second chromosome.
+     */
+    public Chromosome getSecond() {
+        return second;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public String toString() {
+        return String.format("(%s,%s)", getFirst(), getSecond());
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java
new file mode 100755
index 0000000..fa287cf
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Policy used to create a pair of new chromosomes by performing a crossover
+ * operation on a source pair of chromosomes.
+ *
+ */
+public interface CrossoverPolicy {
+
+    /**
+     * Perform a crossover operation on the given chromosomes.
+     *
+     * @param first the first chromosome.
+     * @param second the second chromosome.
+     * @return the pair of new chromosomes that resulted from the crossover.
+     * @throws IllegalArgumentException if the given chromosomes are not compatible with this {@link CrossoverPolicy}
+     */
+    ChromosomePair crossover(Chromosome first, Chromosome second) throws IllegalArgumentException;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java
new file mode 100755
index 0000000..f66e941
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.math3.exception.NotPositiveException;
+import org.apache.commons.math3.exception.NullArgumentException;
+import org.apache.commons.math3.exception.NumberIsTooLargeException;
+import org.apache.commons.math3.exception.OutOfRangeException;
+import org.apache.commons.math3.exception.util.LocalizedFormats;
+import org.apache.commons.math3.util.FastMath;
+
+/**
+ * Population of chromosomes which uses elitism (certain percentage of the best
+ * chromosomes is directly copied to the next generation).
+ *
+ */
+public class ElitisticListPopulation extends ListPopulation {
+
+    /** percentage of chromosomes copied to the next generation */
+    private double elitismRate = 0.9;
+
+    /**
+     * Creates a new {@link ElitisticListPopulation} instance.
+     *
+     * @param chromosomes list of chromosomes in the population
+     * @param populationLimit maximal size of the population
+     * @param elitismRate how many best chromosomes will be directly transferred to the next generation [in %]
+     * @throws NullArgumentException if the list of chromosomes is {@code null}
+     * @throws NotPositiveException if the population limit is not a positive number (&lt; 1)
+     * @throws NumberIsTooLargeException if the list of chromosomes exceeds the population limit
+     * @throws OutOfRangeException if the elitism rate is outside the [0, 1] range
+     */
+    public ElitisticListPopulation(final List<Chromosome> chromosomes, final int populationLimit,
+            final double elitismRate)
+            throws NullArgumentException, NotPositiveException, NumberIsTooLargeException, OutOfRangeException {
+
+        super(chromosomes, populationLimit);
+        setElitismRate(elitismRate);
+    }
+
+    /**
+     * Creates a new {@link ElitisticListPopulation} instance and initializes its inner chromosome list.
+     *
+     * @param populationLimit maximal size of the population
+     * @param elitismRate how many best chromosomes will be directly transferred to the next generation [in %]
+     * @throws NotPositiveException if the population limit is not a positive number (&lt; 1)
+     * @throws OutOfRangeException if the elitism rate is outside the [0, 1] range
+     */
+    public ElitisticListPopulation(final int populationLimit, final double elitismRate)
+            throws NotPositiveException, OutOfRangeException {
+
+        super(populationLimit);
+        setElitismRate(elitismRate);
+    }
+
+    /**
+     * Start the population for the next generation. The <code>{@link #elitismRate}</code>
+     * percents of the best chromosomes are directly copied to the next generation.
+     *
+     * @return the beginnings of the next generation.
+     */
+    public Population nextGeneration() {
+        // initialize a new generation with the same parameters
+        ElitisticListPopulation nextGeneration = new ElitisticListPopulation(getPopulationLimit(), getElitismRate());
+
+        final List<Chromosome> oldChromosomes = getChromosomeList();
+        Collections.sort(oldChromosomes);
+
+        // index of the last "not good enough" chromosome
+        int boundIndex = (int) FastMath.ceil((1.0 - getElitismRate()) * oldChromosomes.size());
+        for (int i = boundIndex; i < oldChromosomes.size(); i++) {
+            nextGeneration.addChromosome(oldChromosomes.get(i));
+        }
+        return nextGeneration;
+    }
+
+    /**
+     * Access the elitism rate.
+     * @return the elitism rate
+     */
+    public double getElitismRate() {
+        return this.elitismRate;
+    }
+
+    /**
+     * Sets the elitism rate, i.e. how many best chromosomes will be directly transferred to the next generation [in %].
+     *
+     * @param elitismRate how many best chromosomes will be directly transferred to the next generation [in %]
+     * @throws OutOfRangeException if the elitism rate is outside the [0, 1] range
+     */
+    public void setElitismRate(final double elitismRate) throws OutOfRangeException {
+        if (elitismRate < 0 || elitismRate > 1) {
+            throw new OutOfRangeException(LocalizedFormats.ELITISM_RATE, elitismRate, 0, 1);
+        }
+        this.elitismRate = elitismRate;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java
new file mode 100755
index 0000000..4388e12
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Fitness of a chromosome.
+ *
+ */
+public interface Fitness {
+
+    /**
+     * Compute the fitness.
+     * @return fitness
+     */
+    double fitness();
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java
new file mode 100755
index 0000000..c0c8316
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import org.apache.commons.math3.exception.NumberIsTooSmallException;
+
+/**
+ * Stops after a fixed number of generations. Each time {@link #isSatisfied(Population)} is invoked, a generation
+ * counter is incremented. Once the counter reaches the configured <code>maxGenerations</code> value,
+ * {@link #isSatisfied(Population)} returns true.
+ *
+ */
+public class FixedGenerationCount implements StoppingCondition {
+    /** Maximum number of generations (stopping criteria) */
+    private final int maxGenerations;
+    /** Number of generations that have passed */
+    private int numGenerations = 0;
+
+    /**
+     * Create a new FixedGenerationCount instance.
+     *
+     * @param maxGenerations number of generations to evolve
+     * @throws NumberIsTooSmallException if the number of generations is &lt; 1
+     */
+    public FixedGenerationCount(final int maxGenerations) throws NumberIsTooSmallException {
+        if (maxGenerations <= 0) {
+            throw new NumberIsTooSmallException(maxGenerations, 1, true);
+        }
+        this.maxGenerations = maxGenerations;
+    }
+
+    /**
+     * Determine whether or not the given number of generations have passed. Increments the number of generations
+     * counter if the maximum has not been reached.
+     *
+     * @param population ignored (no impact on result)
+     * @return <code>true</code> IFF the maximum number of generations has been exceeded
+     */
+    public boolean isSatisfied(final Population population) {
+        if (this.numGenerations < this.maxGenerations) {
+            numGenerations++;
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Returns the number of generations that have already passed.
+     * @return the number of generations that have passed
+     */
+    public int getNumGenerations() {
+        return numGenerations;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java
new file mode 100755
index 0000000..c146274
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.math3.exception.NotPositiveException;
+import org.apache.commons.math3.exception.NullArgumentException;
+import org.apache.commons.math3.exception.NumberIsTooLargeException;
+import org.apache.commons.math3.exception.NumberIsTooSmallException;
+import org.apache.commons.math3.exception.util.LocalizedFormats;
+
+/**
+ * Population of chromosomes represented by a {@link List}.
+ *
+ */
+public abstract class ListPopulation implements Population {
+
+    /** List of chromosomes */
+    private List<Chromosome> chromosomes;
+
+    /** maximal size of the population */
+    private int populationLimit;
+
+    /**
+     * Creates a new ListPopulation instance and initializes its inner chromosome list.
+     *
+     * @param populationLimit maximal size of the population
+     * @throws NotPositiveException if the population limit is not a positive number (&lt; 1)
+     */
+    public ListPopulation(final int populationLimit) throws NotPositiveException {
+        this(Collections.<Chromosome> emptyList(), populationLimit);
+    }
+
+    /**
+     * Creates a new ListPopulation instance.
+     * <p>
+     * Note: the chromosomes of the specified list are added to the population.
+     *
+     * @param chromosomes list of chromosomes to be added to the population
+     * @param populationLimit maximal size of the population
+     * @throws NullArgumentException if the list of chromosomes is {@code null}
+     * @throws NotPositiveException if the population limit is not a positive number (&lt; 1)
+     * @throws NumberIsTooLargeException if the list of chromosomes exceeds the population limit
+     */
+    public ListPopulation(final List<Chromosome> chromosomes, final int populationLimit)
+            throws NullArgumentException, NotPositiveException, NumberIsTooLargeException {
+
+        if (chromosomes == null) {
+            throw new NullArgumentException();
+        }
+        if (populationLimit <= 0) {
+            throw new NotPositiveException(LocalizedFormats.POPULATION_LIMIT_NOT_POSITIVE, populationLimit);
+        }
+        if (chromosomes.size() > populationLimit) {
+            throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+                    chromosomes.size(), populationLimit, false);
+        }
+        this.populationLimit = populationLimit;
+        this.chromosomes = new ArrayList<Chromosome>(populationLimit);
+        this.chromosomes.addAll(chromosomes);
+    }
+
+    /**
+     * Add a {@link Collection} of chromosomes to this {@link Population}.
+     * @param chromosomeColl a {@link Collection} of chromosomes
+     * @throws NumberIsTooLargeException if the population would exceed the population limit when
+     * adding this chromosome
+     * @since 3.1
+     */
+    public void addChromosomes(final Collection<Chromosome> chromosomeColl) throws NumberIsTooLargeException {
+        if (chromosomes.size() + chromosomeColl.size() > populationLimit) {
+            throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+                    chromosomes.size(), populationLimit, false);
+        }
+        this.chromosomes.addAll(chromosomeColl);
+    }
+
+    /**
+     * Returns an unmodifiable list of the chromosomes in this population.
+     * @return the unmodifiable list of chromosomes
+     */
+    public List<Chromosome> getChromosomes() {
+        return Collections.unmodifiableList(chromosomes);
+    }
+
+    /**
+     * Sets the list of chromosomes.
+     * <p>
+     * Note: this method removes all existing chromosomes in the population and adds all chromosomes
+     * of the specified list to the population.
+     *
+     * @param chromosomes the list of chromosomes
+     * @throws NullArgumentException if the list of chromosomes is {@code null}
+     * @throws NumberIsTooLargeException if the list of chromosomes exceeds the population limit
+     * @deprecated use {@link #addChromosomes(Collection)} instead
+     */
+    @Deprecated
+    public void setChromosomes(final List<Chromosome> chromosomes)
+            throws NullArgumentException, NumberIsTooLargeException {
+
+        if (chromosomes == null) {
+            throw new NullArgumentException();
+        }
+        if (chromosomes.size() > populationLimit) {
+            throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+                    chromosomes.size(), populationLimit, false);
+        }
+        this.chromosomes.clear();
+        this.chromosomes.addAll(chromosomes);
+    }
+
+    /**
+     * Access the list of chromosomes.
+     * @return the list of chromosomes
+     * @since 3.1
+     */
+    protected List<Chromosome> getChromosomeList() {
+        return chromosomes;
+    }
+
+    /**
+     * Add the given chromosome to the population.
+     *
+     * @param chromosome the chromosome to add.
+     * @throws NumberIsTooLargeException if the population would exceed the {@code populationLimit} after
+     *   adding this chromosome
+     */
+    public void addChromosome(final Chromosome chromosome) throws NumberIsTooLargeException {
+        if (chromosomes.size() >= populationLimit) {
+            throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+                    chromosomes.size(), populationLimit, false);
+        }
+        this.chromosomes.add(chromosome);
+    }
+
+    /**
+     * Access the fittest chromosome in this population.
+     * @return the fittest chromosome.
+     */
+    public Chromosome getFittestChromosome() {
+        // best so far
+        Chromosome bestChromosome = this.chromosomes.get(0);
+        for (Chromosome chromosome : this.chromosomes) {
+            if (chromosome.compareTo(bestChromosome) > 0) {
+                // better chromosome found
+                bestChromosome = chromosome;
+            }
+        }
+        return bestChromosome;
+    }
+
+    /**
+     * Access the maximum population size.
+     * @return the maximum population size.
+     */
+    public int getPopulationLimit() {
+        return this.populationLimit;
+    }
+
+    /**
+     * Sets the maximal population size.
+     * @param populationLimit maximal population size.
+     * @throws NotPositiveException if the population limit is not a positive number (&lt; 1)
+     * @throws NumberIsTooSmallException if the new population size is smaller than the current number
+     *   of chromosomes in the population
+     */
+    public void setPopulationLimit(final int populationLimit) throws NotPositiveException, NumberIsTooSmallException {
+        if (populationLimit <= 0) {
+            throw new NotPositiveException(LocalizedFormats.POPULATION_LIMIT_NOT_POSITIVE, populationLimit);
+        }
+        if (populationLimit < chromosomes.size()) {
+            throw new NumberIsTooSmallException(populationLimit, chromosomes.size(), true);
+        }
+        this.populationLimit = populationLimit;
+    }
+
+    /**
+     * Access the current population size.
+     * @return the current population size.
+     */
+    public int getPopulationSize() {
+        return this.chromosomes.size();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String toString() {
+        return this.chromosomes.toString();
+    }
+
+    /**
+     * Returns an iterator over the unmodifiable list of chromosomes.
+     * <p>Any call to {@link Iterator#remove()} will result in a {@link UnsupportedOperationException}.</p>
+     *
+     * @return chromosome iterator
+     */
+    public Iterator<Chromosome> iterator() {
+        return getChromosomes().iterator();
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java
new file mode 100755
index 0000000..99ba876
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Algorithm used to mutate a chromosome.
+ *
+ */
+public interface MutationPolicy {
+
+    /**
+     * Mutate the given chromosome.
+     * @param original the original chromosome.
+     * @return the mutated chromosome.
+     * @throws IllegalArgumentException if the given chromosome is not compatible with this {@link MutationPolicy}
+     */
+    Chromosome mutate(Chromosome original) throws IllegalArgumentException;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java
new file mode 100755
index 0000000..02617b8
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.BitSet;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.BitsChromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+
+/**
+ * One point crossover policy. A random crossover point is selected and the
+ * first part from each parent is copied to the corresponding child, and the
+ * second parts are copied crosswise.
+ *
+ * Example:
+ * <pre>
+ * -C- denotes a crossover point
+ *                   -C-                                 -C-
+ * p1 = (1 0 1 0 0 1  | 0 1 1)    X    p2 = (0 1 1 0 1 0  | 1 1 1)
+ *      \------------/ \-----/              \------------/ \-----/
+ *            ||         (*)                       ||        (**)
+ *            VV         (**)                      VV        (*)
+ *      /------------\ /-----\              /------------\ /-----\
+ * c1 = (1 0 1 0 0 1  | 1 1 1)    X    c2 = (0 1 1 0 1 0  | 0 1 1)
+ * </pre>
+ *
+ * This policy works only on {@link BitsChromosome}, and therefore it
+ * is parameterized by T. Moreover, the chromosomes must have same lengths.
+ *
+ *
+ */
+public class OnePointCrossover implements CrossoverPolicy {
+
+    /**
+     * Performs one point crossover. A random crossover point is selected and the
+     * first part from each parent is copied to the corresponding child, and the
+     * second parts are copied crosswise.
+     *
+     * Example:
+     * <pre>
+     * -C- denotes a crossover point
+     *                   -C-                                 -C-
+     * p1 = (1 0 1 0 0 1  | 0 1 1)    X    p2 = (0 1 1 0 1 0  | 1 1 1)
+     *      \------------/ \-----/              \------------/ \-----/
+     *            ||         (*)                       ||        (**)
+     *            VV         (**)                      VV        (*)
+     *      /------------\ /-----\              /------------\ /-----\
+     * c1 = (1 0 1 0 0 1  | 1 1 1)    X    c2 = (0 1 1 0 1 0  | 0 1 1)
+     * </pre>
+     *
+     * @param first first parent (p1)
+     * @param second second parent (p2)
+     * @return pair of two children (c1,c2)
+     * @throws IllegalArgumentException if one of the chromosomes is
+     *   not an instance of {@link BitsChromosome}
+     * @throws ChromosomeMismatchException if the length of the two chromosomes is different
+     */
+    @SuppressWarnings("unchecked") // OK because of instanceof checks
+    public ChromosomePair crossover(final Chromosome first, final Chromosome second)
+            throws IllegalArgumentException, ChromosomeMismatchException {
+
+        if (!(first instanceof BitsChromosome && second instanceof BitsChromosome)) {
+            throw new IllegalArgumentException("Chromosome must be of type BitsChromosome.");
+        }
+        return crossover((BitsChromosome) first, (BitsChromosome) second);
+    }
+
+    /**
+     * Helper for {@link #crossover(Chromosome, Chromosome)}. Performs the actual crossover.
+     *
+     * @param first the first chromosome.
+     * @param second the second chromosome.
+     * @return the pair of new chromosomes that resulted from the crossover.
+     * @throws ChromosomeMismatchException if the length of the two chromosomes is different
+     */
+    private ChromosomePair crossover(final BitsChromosome first, final BitsChromosome second)
+            throws ChromosomeMismatchException {
+        final int length = first.getLength();
+        if (length != second.getLength()) {
+            throw new ChromosomeMismatchException("BitsChromosome length mismatch.", second.getLength(), length);
+        }
+
+        final BitSet parent1Key = first.getKey();
+        final BitSet parent2Key = second.getKey();
+
+        final BitSet child1Key = new BitSet(length);
+        final BitSet child2Key = new BitSet(length);
+
+        // select a crossover point at random (0 and length makes no sense)
+        final int crossoverIndex = 1 + (GeneticAlgorithm.RANDGEN.get().nextInt(length - 2));
+
+        BitSet a = (BitSet) parent1Key.clone();
+        a.clear(crossoverIndex, length);
+        BitSet b = (BitSet) parent2Key.clone();
+        b.clear(0, crossoverIndex);
+
+        BitSet c = (BitSet) parent1Key.clone();
+        c.clear(crossoverIndex, length);
+        BitSet d = (BitSet) parent2Key.clone();
+        d.clear(0, crossoverIndex);
+
+        child1Key.or(a);
+        child1Key.or(d);
+
+        child2Key.or(c);
+        child2Key.or(b);
+        return new ChromosomePair(first.newBitsChromosome(child1Key), second.newBitsChromosome(child2Key));
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java
new file mode 100755
index 0000000..8e35420
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import org.apache.commons.math3.exception.NumberIsTooLargeException;
+
+/**
+ * A collection of chromosomes that facilitates generational evolution.
+ *
+ */
+public interface Population extends Iterable<Chromosome> {
+    /**
+     * Access the current population size.
+     * @return the current population size.
+     */
+    int getPopulationSize();
+
+    /**
+     * Access the maximum population size.
+     * @return the maximum population size.
+     */
+    int getPopulationLimit();
+
+    /**
+     * Start the population for the next generation.
+     * @return the beginnings of the next generation.
+     */
+    Population nextGeneration();
+
+    /**
+     * Add the given chromosome to the population.
+     * @param chromosome the chromosome to add.
+     * @throws NumberIsTooLargeException if the population would exceed the population limit when adding
+     *   this chromosome
+     */
+    void addChromosome(Chromosome chromosome) throws NumberIsTooLargeException;
+
+    /**
+     * Access the fittest chromosome in this population.
+     * @return the fittest chromosome.
+     */
+    Chromosome getFittestChromosome();
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java
new file mode 100755
index 0000000..1c32f88
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Algorithm used to select a chromosome pair from a population.
+ *
+ */
+public interface SelectionPolicy {
+    /**
+     * Select two chromosomes from the population.
+     * @param population the population from which the chromosomes are choosen.
+     * @return the selected chromosomes.
+     * @throws IllegalArgumentException if the population is not compatible with this {@link SelectionPolicy}
+     */
+    ChromosomePair select(Population population) throws IllegalArgumentException;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java
new file mode 100755
index 0000000..28b879a
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Algorithm used to determine when to stop evolution.
+ *
+ */
+public interface StoppingCondition {
+    /**
+     * Determine whether or not the given population satisfies the stopping condition.
+     *
+     * @param population the population to test.
+     * @return <code>true</code> if this stopping condition is met by the given population,
+     *   <code>false</code> otherwise.
+     */
+    boolean isSatisfied(Population population);
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java
new file mode 100755
index 0000000..cca44ca
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Tournament selection scheme. Each of the two selected chromosomes is selected
+ * based on n-ary tournament -- this is done by drawing {@link #arity} random
+ * chromosomes without replacement from the population, and then selecting the
+ * fittest chromosome among them.
+ *
+ */
+public class TournamentSelection implements SelectionPolicy {
+
+    /** number of chromosomes included in the tournament selections */
+    private int arity;
+
+    /**
+     * Creates a new TournamentSelection instance.
+     *
+     * @param arity how many chromosomes will be drawn to the tournament
+     */
+    public TournamentSelection(final int arity) {
+        this.arity = arity;
+    }
+
+    /**
+     * Select two chromosomes from the population. Each of the two selected
+     * chromosomes is selected based on n-ary tournament -- this is done by
+     * drawing {@link #arity} random chromosomes without replacement from the
+     * population, and then selecting the fittest chromosome among them.
+     *
+     * @param population the population from which the chromosomes are chosen.
+     * @return the selected chromosomes.
+     * @throws IllegalArgumentException if the tournament arity is bigger than the population size
+     */
+    public ChromosomePair select(final Population population) throws IllegalArgumentException {
+        return new ChromosomePair(tournament((ListPopulation) population), tournament((ListPopulation) population));
+    }
+
+    /**
+     * Helper for {@link #select(Population)}. Draw {@link #arity} random chromosomes without replacement from the
+     * population, and then select the fittest chromosome among them.
+     *
+     * @param population the population from which the chromosomes are chosen.
+     * @return the selected chromosome.
+     * @throws IllegalArgumentException if the tournament arity is bigger than the population size
+     */
+    private Chromosome tournament(final ListPopulation population) throws IllegalArgumentException {
+        if (population.getPopulationSize() < this.arity) {
+            throw new IllegalArgumentException("Tournament arty is too large.");
+        }
+        // auxiliary population
+        ListPopulation tournamentPopulation = new ListPopulation(this.arity) {
+            /** {@inheritDoc} */
+            public Population nextGeneration() {
+                // not useful here
+                return null;
+            }
+        };
+
+        // create a copy of the chromosome list
+        List<Chromosome> chromosomes = Lists.newArrayList(population.getChromosomes());
+        for (int i = 0; i < this.arity; i++) {
+            // select a random individual and add it to the tournament
+            int rind = GeneticAlgorithm.RANDGEN.get().nextInt(chromosomes.size());
+            tournamentPopulation.addChromosome(chromosomes.get(rind));
+            // do not select it again
+            chromosomes.remove(rind);
+        }
+        // the winner takes it all
+        return tournamentPopulation.getFittestChromosome();
+    }
+
+    /**
+     * Gets the arity (number of chromosomes drawn to the tournament).
+     *
+     * @return arity of the tournament
+     */
+    public int getArity() {
+        return arity;
+    }
+
+    /**
+     * Sets the arity (number of chromosomes drawn to the tournament).
+     *
+     * @param arity arity of the tournament
+     */
+    public void setArity(final int arity) {
+        this.arity = arity;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3c4c56e/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java
new file mode 100755
index 0000000..4cefbd4
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+import org.junit.Test;
+
+public class GeneticAlgorithmTest extends AlgorithmTestBase {
+
+    @Test
+    public void testBPUSCalculator() {
+        BenefitPolicy benefitPolicy = new BPUSCalculator(cuboidStats);
+        GeneticAlgorithm algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+
+        List<Long> recommendList = algorithm.recommend(10);
+        System.out.println("recommendList by BPUSCalculator: " + recommendList);
+        System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+    }
+
+    @Test
+    public void testPBPUSCalculator() {
+        BenefitPolicy benefitPolicy = new PBPUSCalculator(cuboidStats);
+        GeneticAlgorithm algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+
+        List<Long> recommendList = algorithm.recommend(10);
+        System.out.println("recommendList by PBPUSCalculator:" + recommendList);
+        System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+    }
+
+    @Test
+    public void testSPBPUSCalculator() {
+        BenefitPolicy benefitPolicy = new SPBPUSCalculator(cuboidStats);
+        GeneticAlgorithm algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+
+        List<Long> recommendList = algorithm.recommend(10);
+        System.out.println("recommendList by SPBPUSCalculator:" + recommendList);
+        System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+    }
+}


[14/22] kylin git commit: APACHE-KYLIN-2783: change the cuboid scheduler for FactDistinctColumnsMapper

Posted by nj...@apache.org.
APACHE-KYLIN-2783: change the cuboid scheduler for FactDistinctColumnsMapper


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74f581e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74f581e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74f581e7

Branch: refs/heads/yaho-cube-planner
Commit: 74f581e78e491306d151545c78d717bf0bce79f8
Parents: 386a2ff
Author: Zhong <nj...@apache.org>
Authored: Mon Aug 28 13:41:29 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:04 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/74f581e7/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 16bc03c..4e4c332 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -88,7 +88,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
         collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
         if (collectStatistics) {
             samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
-            cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
+            cuboidScheduler = cubeSeg.getCuboidScheduler();
             nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
 
             List<Long> cuboidIdList = Lists.newArrayList();


[07/22] kylin git commit: APACHE-KYLIN-2783: Refactor CuboidScheduler to be extensible

Posted by nj...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 544e072..12446ff 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -201,7 +201,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
                         if (initialized == false) {
                             prepare();
                             long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-                            Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+                                    Cuboid baseCuboid = Cuboid.findById(cubeSegment, baseCuboidId);
                             baseCuboidBuilder = new BaseCuboidBuilder(kylinConfig, cubeDesc, cubeSegment, intermediateTableDesc, AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid), MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
                             initialized = true;
                         }
@@ -244,7 +244,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
             reducerFunction2 = new CuboidReducerFunction2(measureNum, vCubeDesc.getValue(), measureAggregators, needAggr);
         }
 
-        final int totalLevels = cubeDesc.getBuildLevel();
+        final int totalLevels = cubeSegment.getCuboidScheduler().getBuildLevel();
         JavaPairRDD<ByteArray, Object[]>[] allRDDs = new JavaPairRDD[totalLevels + 1];
         int level = 0;
         int partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
@@ -360,9 +360,9 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
 
             byte[] key = tuple2._1().array();
             long cuboidId = rowKeySplitter.split(key);
-            Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
+            Cuboid parentCuboid = Cuboid.findById(cubeSegment, cuboidId);
 
-            Collection<Long> myChildren = cubeDesc.getCuboidScheduler().getSpanningCuboid(cuboidId);
+            Collection<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId);
 
             // if still empty or null
             if (myChildren == null || myChildren.size() == 0) {
@@ -371,7 +371,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
 
             List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size());
             for (Long child : myChildren) {
-                Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
+                Cuboid childCuboid = Cuboid.findById(cubeSegment, child);
                 Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
 
                 byte[] newKey = new byte[result.getFirst()];

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
index 793cd87..d7b20c8 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
@@ -65,7 +65,7 @@ public final class DefaultTupleConverter implements TupleConverter {
 
     @Override
     public final Tuple2<byte[], byte[]> convert(long cuboidId, GTRecord record) {
-        Cuboid cuboid = Cuboid.findById(segment.getCubeDesc(), cuboidId);
+        Cuboid cuboid = Cuboid.findById(segment, cuboidId);
         RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid);
 
         final int dimensions = Long.bitCount(cuboidId);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
index a19b8c7..5e5b16a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -84,7 +84,7 @@ public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
         long randSeed = System.currentTimeMillis();
 
         IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
-        DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
+        DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getCuboidScheduler(), flatDesc, dictionaryMap);
         doggedBuilder.setConcurrentThreads(THREADS);
 
         {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index dbd9ce2..1e10d79 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -88,7 +88,7 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
         long randSeed = System.currentTimeMillis();
 
         IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
-        DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
+        DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getCuboidScheduler(), flatDesc, dictionaryMap);
         doggedBuilder.setConcurrentThreads(THREADS);
         doggedBuilder.setSplitRowThreshold(SPLIT_ROWS);
         FileRecordWriter doggedResult = new FileRecordWriter();
@@ -100,7 +100,7 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
             doggedResult.close();
         }
 
-        InMemCubeBuilder inmemBuilder = new InMemCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
+        InMemCubeBuilder inmemBuilder = new InMemCubeBuilder(cube.getCuboidScheduler(), flatDesc, dictionaryMap);
         inmemBuilder.setConcurrentThreads(THREADS);
         FileRecordWriter inmemResult = new FileRecordWriter();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index ea66c73..49d267a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -109,7 +109,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
     private void testBuildInner() throws Exception {
 
         IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
-        InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
+        InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(cube.getCuboidScheduler(), flatDesc, dictionaryMap);
         //DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
         cubeBuilder.setConcurrentThreads(nThreads);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 6587d4e..59f1349 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -81,7 +81,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
     //TODO:shardingonstreaming
     private byte[] createKey(Long cuboidId, GTRecord record) {
         if (rowKeyEncoder == null || rowKeyEncoder.getCuboidID() != cuboidId) {
-            rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
+            rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeSegment, cuboidId));
             keybuf = rowKeyEncoder.createBuf();
         }
         rowKeyEncoder.encode(record, record.getInfo().getPrimaryKey(), keybuf);
@@ -92,7 +92,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
     @Override
     public void write(long cuboidId, GTRecord record) throws IOException {
         byte[] key = createKey(cuboidId, record);
-        final Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
+        final Cuboid cuboid = Cuboid.findById(cubeSegment, cuboidId);
         final int nDims = cuboid.getColumns().size();
         final ImmutableBitSet bitSet = new ImmutableBitSet(nDims, nDims + cubeDesc.getMeasures().size());
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
index f151876..7d089ac 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
@@ -91,10 +91,10 @@ public class MeasureTypeOnlyAggrInBaseTest extends LocalFileMetadataTestCase {
     @Test
     public void testIdentifyCuboidV2() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException {
         CubeDesc cubeDesc = cube.getDescriptor();
-        Cuboid ret = Cuboid.identifyCuboid(cubeDesc, Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList());
+        Cuboid ret = Cuboid.identifyCuboid(cube, Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList());
         long baseCuboidId = cubeDesc.getRowkey().getFullMask();
         assertNotEquals(baseCuboidId, ret.getId());
-        ret = Cuboid.identifyCuboid(cubeDesc, dimensions, metrics);
+        ret = Cuboid.identifyCuboid(cube, dimensions, metrics);
         assertEquals(baseCuboidId, ret.getId());
     }
 


[18/22] kylin git commit: APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority

Posted by nj...@apache.org.
APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd78cf1f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd78cf1f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd78cf1f

Branch: refs/heads/yaho-cube-planner
Commit: bd78cf1fc89067542eb05161eafab6a4ee97fe99
Parents: ab3ac1a
Author: Zhong <nj...@apache.org>
Authored: Thu Aug 31 14:56:19 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:05 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |   8 ++
 .../kylin/job/engine/JobEngineConfig.java       |  14 +++
 .../kylin/job/execution/AbstractExecutable.java |   9 ++
 .../job/execution/CheckpointExecutable.java     |   7 ++
 .../job/execution/DefaultChainedExecutable.java |   7 ++
 .../job/impl/threadpool/DefaultScheduler.java   | 111 ++++++++++++++++++-
 .../org/apache/kylin/engine/mr/CubingJob.java   |  19 +++-
 7 files changed, 171 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d66d7ce..53bb2ad 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -503,6 +503,14 @@ abstract public class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0"));
     }
 
+    public boolean getSchedulerPriorityConsidered() {
+        return Boolean.parseBoolean(getOptional("kylin.job.scheduler.priority-considered", "false"));
+    }
+
+    public Integer getSchedulerPriorityBarFetchFromQueue() {
+        return Integer.parseInt(getOptional("kylin.job.scheduler.priority-bar-fetch-from-queue", "20"));
+    }
+
     public Integer getSchedulerPollIntervalSecond() {
         return Integer.parseInt(getOptional("kylin.job.scheduler.poll-interval-second", "30"));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 6890557..9ba602f 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -105,6 +105,20 @@ public class JobEngineConfig {
         return config;
     }
 
+    /**
+     * @return if consider job priority when scheduling jobs
+     * */
+    public boolean getJobPriorityConsidered() {
+        return config.getSchedulerPriorityConsidered();
+    }
+
+    /**
+     * @return the priority bar for fetching jobs from job priority queue
+     */
+    public int getFetchQueuePriorityBar() {
+        return config.getSchedulerPriorityBarFetchFromQueue();
+    }
+
     public String getHdfsWorkingDirectory() {
         return config.getHdfsWorkingDirectory();
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 30b6421..a37cdc9 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -44,6 +44,8 @@ import com.google.common.collect.Maps;
  */
 public abstract class AbstractExecutable implements Executable, Idempotent {
 
+    public static final Integer DEFAULT_PRIORITY = 10;
+
     protected static final String SUBMITTER = "submitter";
     protected static final String NOTIFY_LIST = "notify_list";
     protected static final String START_TIME = "startTime";
@@ -389,6 +391,13 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
         return output.getState() == ExecutableState.READY;
     }
 
+    /**
+     * The larger the value, the higher priority
+     * */
+    public int getDefaultPriority() {
+        return DEFAULT_PRIORITY;
+    }
+
     /*
     * discarded is triggered by JobService, the Scheduler is not awake of that
     *

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
index 9864400..db477cb 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
@@ -29,6 +29,8 @@ public class CheckpointExecutable extends DefaultChainedExecutable {
 
     private static final Logger logger = LoggerFactory.getLogger(CheckpointExecutable.class);
 
+    public static final Integer DEFAULT_PRIORITY = 30;
+
     private static final String DEPLOY_ENV_NAME = "envName";
     private static final String PROJECT_INSTANCE_NAME = "projectName";
 
@@ -75,4 +77,9 @@ public class CheckpointExecutable extends DefaultChainedExecutable {
     public void setProjectName(String name) {
         setParam(PROJECT_INSTANCE_NAME, name);
     }
+
+    @Override
+    public int getDefaultPriority() {
+        return DEFAULT_PRIORITY;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 0ade541..8a816ba 100755
--- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -31,6 +31,8 @@ import com.google.common.collect.Maps;
  */
 public class DefaultChainedExecutable extends AbstractExecutable implements ChainedExecutable {
 
+    public static final Integer DEFAULT_PRIORITY = 10;
+
     private final List<AbstractExecutable> subTasks = Lists.newArrayList();
 
     public DefaultChainedExecutable() {
@@ -168,4 +170,9 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
         executable.setId(getId() + "-" + String.format("%02d", subTasks.size()));
         this.subTasks.add(executable);
     }
+
+    @Override
+    public int getDefaultPriority() {
+        return DEFAULT_PRIORITY;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 315671c..684bd5b 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -18,7 +18,9 @@
 
 package org.apache.kylin.job.impl.threadpool;
 
+import java.util.Comparator;
 import java.util.Map;
+import java.util.PriorityQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -29,6 +31,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.state.ConnectionState;
 import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.common.util.SetThreadName;
 import org.apache.kylin.job.Scheduler;
 import org.apache.kylin.job.engine.JobEngineConfig;
@@ -51,7 +54,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
 
     private JobLock jobLock;
     private ExecutableManager executableManager;
-    private FetcherRunner fetcher;
+    private Runnable fetcher;
     private ScheduledExecutorService fetcherPool;
     private ExecutorService jobPool;
     private DefaultContext context;
@@ -69,6 +72,110 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
         }
     }
 
+    private class FetcherRunnerWithPriority implements Runnable {
+        volatile PriorityQueue<Pair<AbstractExecutable, Integer>> jobPriorityQueue = new PriorityQueue<>(1,
+                new Comparator<Pair<AbstractExecutable, Integer>>() {
+                    @Override
+                    public int compare(Pair<AbstractExecutable, Integer> o1, Pair<AbstractExecutable, Integer> o2) {
+                        return o1.getSecond() > o2.getSecond() ? -1 : 1;
+                    }
+                });
+
+        private void addToJobPool(AbstractExecutable executable, int priority) {
+            String jobDesc = executable.toString();
+            logger.info(jobDesc + " prepare to schedule and its priority is " + priority);
+            try {
+                context.addRunningJob(executable);
+                jobPool.execute(new JobRunner(executable));
+                logger.info(jobDesc + " scheduled");
+            } catch (Exception ex) {
+                context.removeRunningJob(executable);
+                logger.warn(jobDesc + " fail to schedule", ex);
+            }
+        }
+
+        @Override
+        synchronized public void run() {
+            try {
+                // logger.debug("Job Fetcher is running...");
+                Map<String, Executable> runningJobs = context.getRunningJobs();
+
+                // fetch job from jobPriorityQueue first to reduce chance to scan job list
+                Map<String, Integer> leftJobPriorities = Maps.newHashMap();
+                Pair<AbstractExecutable, Integer> executableWithPriority;
+                while ((executableWithPriority = jobPriorityQueue.peek()) != null
+                        // the priority of jobs in pendingJobPriorities should be above a threshold
+                        && executableWithPriority.getSecond() >= jobEngineConfig.getFetchQueuePriorityBar()) {
+                    executableWithPriority = jobPriorityQueue.poll();
+                    AbstractExecutable executable = executableWithPriority.getFirst();
+                    int curPriority = executableWithPriority.getSecond();
+                    // the job should wait more than one time
+                    if (curPriority > executable.getDefaultPriority() + 1) {
+                        addToJobPool(executable, curPriority);
+                    } else {
+                        leftJobPriorities.put(executable.getId(), curPriority + 1);
+                    }
+                }
+
+                if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
+                    logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
+                    return;
+                }
+
+                while ((executableWithPriority = jobPriorityQueue.poll()) != null) {
+                    leftJobPriorities.put(executableWithPriority.getFirst().getId(),
+                            executableWithPriority.getSecond() + 1);
+                }
+
+                int nRunning = 0, nReady = 0, nStopped = 0, nOthers = 0, nError = 0, nDiscarded = 0, nSUCCEED = 0;
+                for (final String id : executableManager.getAllJobIds()) {
+                    if (runningJobs.containsKey(id)) {
+                        // logger.debug("Job id:" + id + " is already running");
+                        nRunning++;
+                        continue;
+                    }
+
+                    AbstractExecutable executable = executableManager.getJob(id);
+                    if (!executable.isReady()) {
+                        final Output output = executableManager.getOutput(id);
+                        // logger.debug("Job id:" + id + " not runnable");
+                        if (output.getState() == ExecutableState.DISCARDED) {
+                            nDiscarded++;
+                        } else if (output.getState() == ExecutableState.ERROR) {
+                            nError++;
+                        } else if (output.getState() == ExecutableState.SUCCEED) {
+                            nSUCCEED++;
+                        } else if (output.getState() == ExecutableState.STOPPED) {
+                            nStopped++;
+                        } else {
+                            nOthers++;
+                        }
+                        continue;
+                    }
+
+                    nReady++;
+                    Integer priority = leftJobPriorities.get(id);
+                    if (priority == null) {
+                        priority = executable.getDefaultPriority();
+                    }
+                    jobPriorityQueue.add(new Pair<>(executable, priority));
+                }
+
+                while (runningJobs.size() < jobEngineConfig.getMaxConcurrentJobLimit()
+                        && (executableWithPriority = jobPriorityQueue.poll()) != null) {
+                    addToJobPool(executableWithPriority.getFirst(), executableWithPriority.getSecond());
+                }
+
+                logger.info("Job Fetcher: " + nRunning + " running, " + runningJobs.size() + " actual running, "
+                        + nStopped + " stopped, " + nReady + " ready, " + jobPriorityQueue.size() + " waiting, " //
+                        + nSUCCEED + " already succeed, " + nError + " error, " + nDiscarded + " discarded, " + nOthers
+                        + " others");
+            } catch (Exception e) {
+                logger.warn("Job Fetcher caught a exception " + e);
+            }
+        }
+    }
+
     private class FetcherRunner implements Runnable {
 
         @Override
@@ -218,7 +325,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
 
         int pollSecond = jobEngineConfig.getPollIntervalSecond();
         logger.info("Fetching jobs every {} seconds", pollSecond);
-        fetcher = new FetcherRunner();
+        fetcher = jobEngineConfig.getJobPriorityConsidered() ? new FetcherRunnerWithPriority() : new FetcherRunner();
         fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
         hasStarted = true;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd78cf1f/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 1fa56c4..056b943 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -65,12 +65,18 @@ public class CubingJob extends DefaultChainedExecutable {
     }
 
     public enum CubingJobTypeEnum {
-        BUILD("BUILD"), OPTIMIZE("OPTIMIZE"), MERGE("MERGE");
+        BUILD("BUILD", 20), OPTIMIZE("OPTIMIZE", 5), MERGE("MERGE", 25);
 
         private final String name;
+        private final int defaultPriority;
 
-        CubingJobTypeEnum(String name) {
+        CubingJobTypeEnum(String name, int priority) {
             this.name = name;
+            this.defaultPriority = priority;
+        }
+
+        public int getDefaultPriority() {
+            return defaultPriority;
         }
 
         public String toString() {
@@ -151,6 +157,15 @@ public class CubingJob extends DefaultChainedExecutable {
         super();
     }
 
+    @Override
+    public int getDefaultPriority() {
+        CubingJobTypeEnum jobType = CubingJobTypeEnum.getByName(getJobType());
+        if (jobType == null) {
+            return super.getDefaultPriority();
+        }
+        return jobType.getDefaultPriority();
+    }
+
     protected void setDeployEnvName(String name) {
         setParam(DEPLOY_ENV_NAME, name);
     }


[06/22] kylin git commit: APACHE-KYLIN-2728: Introduce a new cuboid scheduler based on cuboid tree rather than static rules

Posted by nj...@apache.org.
APACHE-KYLIN-2728: Introduce a new cuboid scheduler based on cuboid tree rather than static rules


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9b67d58c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9b67d58c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9b67d58c

Branch: refs/heads/yaho-cube-planner
Commit: 9b67d58cdba820eba688fed53e687970d6ce13cc
Parents: 2425bf0
Author: Zhong <nj...@apache.org>
Authored: Thu Aug 17 20:40:35 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:38 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/CubeInstance.java     | 132 +++++++-
 .../kylin/cube/cuboid/CuboidModeEnum.java       |  48 +++
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  | 313 +++++++++++++++++++
 .../cube/cuboid/TreeCuboidSchedulerTest.java    | 148 +++++++++
 4 files changed, 638 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9b67d58c/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 6369b38..3f8bbbe 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,16 +18,26 @@
 
 package org.apache.kylin.cube;
 
+import static org.apache.kylin.cube.cuboid.CuboidModeEnum.CURRENT;
+import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND;
+
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -94,6 +104,15 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
     @JsonProperty("create_time_utc")
     private long createTimeUTC;
 
+    @JsonProperty("cuboidBytes")
+    private byte[] cuboidBytes;
+
+    @JsonProperty("cuboid_bytes_recommend")
+    private byte[] cuboidBytesRecommend;
+
+    @JsonProperty("last_optimized")
+    private long lastOptimized;
+
     // cuboid scheduler lazy built
     transient private CuboidScheduler cuboidScheduler;
 
@@ -107,7 +126,14 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
 
         synchronized (this) {
             if (cuboidScheduler == null) {
-                cuboidScheduler = getDescriptor().getInitialCuboidScheduler();
+                Map<Long, Long> cuboidsWithRowCnt = getCuboids();
+                if (cuboidsWithRowCnt == null) {
+                    cuboidScheduler = getDescriptor().getInitialCuboidScheduler();
+                } else {
+                    cuboidScheduler = new TreeCuboidScheduler(getDescriptor(),
+                            Lists.newArrayList(cuboidsWithRowCnt.keySet()),
+                            new TreeCuboidScheduler.CuboidCostComparator(cuboidsWithRowCnt));
+                }
             }
         }
         return cuboidScheduler;
@@ -143,7 +169,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
     // in a temporary broken state, so that user can edit and fix it. Broken state is often due to
     // schema changes at source.
     public boolean allowBrokenDescriptor() {
-        return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
+        return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN)
+                && segments.isEmpty();
     }
 
     public String getResourcePath() {
@@ -303,6 +330,104 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         this.createTimeUTC = createTimeUTC;
     }
 
+    public Set<Long> getCuboidsByMode(String cuboidModeName) {
+        return getCuboidsByMode(CuboidModeEnum.getByModeName(cuboidModeName));
+    }
+
+    public Set<Long> getCuboidsByMode(CuboidModeEnum cuboidMode) {
+        if (cuboidMode == null || cuboidMode == CURRENT) {
+            return getCuboidScheduler().getAllCuboidIds();
+        }
+        Set<Long> cuboidsRecommend = getCuboidsRecommend();
+        if (cuboidsRecommend == null || cuboidMode == RECOMMEND) {
+            return cuboidsRecommend;
+        }
+        Set<Long> currentCuboids = getCuboidScheduler().getAllCuboidIds();
+        switch (cuboidMode) {
+        case RECOMMEND_EXISTING:
+            cuboidsRecommend.retainAll(currentCuboids);
+            return cuboidsRecommend;
+        case RECOMMEND_MISSING:
+            cuboidsRecommend.removeAll(currentCuboids);
+            return cuboidsRecommend;
+        case RECOMMEND_MISSING_WITH_BASE:
+            cuboidsRecommend.removeAll(currentCuboids);
+            currentCuboids.add(getCuboidScheduler().getBaseCuboidId());
+            return cuboidsRecommend;
+        default:
+            return null;
+        }
+    }
+
+    public HashMap<Long, Long> getCuboids() {
+        if (cuboidBytes == null)
+            return null;
+        byte[] uncompressed;
+        try {
+            uncompressed = CompressionUtils.decompress(cuboidBytes);
+            String str = new String(uncompressed, "UTF-8");
+            HashMap<Long, Long> cuboids = JsonUtil.readValue(str, HashMap.class);
+            return cuboids.isEmpty() ? null : cuboids;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void setCuboids(HashMap<Long, Long> cuboids) {
+        if (cuboids == null)
+            return;
+        if (cuboids.isEmpty()) {
+            cuboidBytes = null;
+            return;
+        }
+
+        try {
+            String str = JsonUtil.writeValueAsString(cuboids);
+            byte[] compressed = CompressionUtils.compress(str.getBytes("UTF-8"));
+            cuboidBytes = compressed;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public HashSet<Long> getCuboidsRecommend() {
+        if (cuboidBytesRecommend == null)
+            return null;
+        byte[] uncompressed;
+        try {
+            uncompressed = CompressionUtils.decompress(cuboidBytesRecommend);
+            String str = new String(uncompressed, "UTF-8");
+            HashSet<Long> cuboids = JsonUtil.readValue(str, HashSet.class);
+            return cuboids.isEmpty() ? null : cuboids;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void setCuboidsRecommend(HashSet<Long> cuboids) {
+        if (cuboids == null)
+            return;
+        if (cuboids.isEmpty()) {
+            cuboidBytesRecommend = null;
+            return;
+        }
+        try {
+            String str = JsonUtil.writeValueAsString(cuboids);
+            byte[] compressed = CompressionUtils.compress(str.getBytes("UTF-8"));
+            cuboidBytesRecommend = compressed;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Get cuboid level count except base cuboid
+     * @return
+     */
+    public int getBuildLevel() {
+        return getCuboidScheduler().getCuboidsByLayer().size() - 1;
+    }
+
     @Override
     public CapabilityResult isCapable(SQLDigest digest) {
         CapabilityResult result = CubeCapabilityChecker.check(this, digest);
@@ -377,7 +502,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
             return false;
 
-        return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
+        return this.getDescriptor().getAutoMergeTimeRanges() != null
+                && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
     }
 
     public Pair<Long, Long> autoMergeCubeSegments() throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9b67d58c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidModeEnum.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidModeEnum.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidModeEnum.java
new file mode 100644
index 0000000..f55c9db
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidModeEnum.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import com.google.common.base.Strings;
+
+public enum CuboidModeEnum {
+    CURRENT("CURRENT"), RECOMMEND("RECOMMEND"), RECOMMEND_EXISTING("RECOMMEND_EXISTING"), RECOMMEND_MISSING(
+            "RECOMMEND_MISSING"), RECOMMEND_MISSING_WITH_BASE("RECOMMEND_MISSING_WITH_BASE");
+
+    private final String modeName;
+
+    CuboidModeEnum(String modeName) {
+        this.modeName = modeName;
+    }
+
+    public String toString() {
+        return modeName;
+    }
+
+    public static CuboidModeEnum getByModeName(String modeName) {
+        if (Strings.isNullOrEmpty(modeName)) {
+            return null;
+        }
+        for (CuboidModeEnum mode : CuboidModeEnum.values()) {
+            if (mode.modeName.equals(modeName.toUpperCase())) {
+                return mode;
+            }
+        }
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9b67d58c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
new file mode 100644
index 0000000..73f4824
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.model.CubeDesc;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class TreeCuboidScheduler extends CuboidScheduler {
+
+    final private CuboidTree cuboidTree;
+
+    public TreeCuboidScheduler(CubeDesc cubeDesc, List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+        super(cubeDesc);
+        cuboidTree = CuboidTree.createFromCuboids(allCuboidIds, cuboidComparator);
+    }
+
+    @Override
+    public Set<Long> getAllCuboidIds() {
+        return cuboidTree.getAllCuboidIds();
+    }
+
+    @Override
+    public int getCuboidCount() {
+        return cuboidTree.getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc));
+    }
+
+    @Override
+    public List<Long> getSpanningCuboid(long cuboidId) {
+        return cuboidTree.getSpanningCuboid(cuboidId);
+    }
+
+    @Override
+    public long findBestMatchCuboid(long cuboidId) {
+        return cuboidTree.findBestMatchCuboid(cuboidId);
+    }
+
+    public static class CuboidTree {
+        private int treeLevels;
+
+        private TreeNode root;
+
+        private Comparator<Long> cuboidComparator;
+
+        private Map<Long, TreeNode> index = new HashMap<>();
+
+        @VisibleForTesting
+        static CuboidTree createFromCuboids(List<Long> allCuboidIds) {
+            return createFromCuboids(allCuboidIds, Cuboid.cuboidSelectComparator);
+        }
+
+        @VisibleForTesting
+        static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+            // sort the cuboid ids in descending order, so that don't need to adjust
+            // the cuboid tree when adding cuboid id to the tree.
+            Collections.sort(allCuboidIds, new Comparator<Long>() {
+                @Override
+                public int compare(Long o1, Long o2) {
+                    return Long.compare(o2, o1);
+                }
+            });
+            long basicCuboidId = allCuboidIds.get(0);
+            CuboidTree cuboidTree = new CuboidTree(cuboidComparator);
+            cuboidTree.setRoot(basicCuboidId);
+
+            for (long cuboidId : allCuboidIds) {
+                cuboidTree.addCuboid(cuboidId);
+            }
+            cuboidTree.buildIndex();
+            return cuboidTree;
+        }
+
+        private CuboidTree(Comparator<Long> cuboidComparator) {
+            this.cuboidComparator = cuboidComparator;
+        }
+
+        public Set<Long> getAllCuboidIds() {
+            return index.keySet();
+        }
+
+        public List<Long> getSpanningCuboid(long cuboidId) {
+            TreeNode node = index.get(cuboidId);
+            if (node == null) {
+                throw new IllegalArgumentException("the cuboid:" + cuboidId + " is not exist in the tree");
+            }
+
+            return Lists.transform(node.children, new Function<TreeNode, Long>() {
+                @Nullable
+                @Override
+                public Long apply(@Nullable TreeNode input) {
+                    return input.cuboidId;
+                }
+            });
+        }
+
+        public long findBestMatchCuboid(long cuboidId) {
+            // exactly match
+            if (index.containsKey(cuboidId)) {
+                return cuboidId;
+            }
+
+            return findBestParent(cuboidId).cuboidId;
+        }
+
+        private int getCuboidCount(long cuboidId) {
+            int r = 1;
+            for (Long child : getSpanningCuboid(cuboidId)) {
+                r += getCuboidCount(child);
+            }
+            return r;
+        }
+
+        public void print(PrintWriter out) {
+            int dimensionCnt = Long.bitCount(root.cuboidId);
+            doPrint(root, dimensionCnt, 0, out);
+        }
+
+        private void doPrint(TreeNode node, int dimensionCount, int depth, PrintWriter out) {
+            printCuboid(node.cuboidId, dimensionCount, depth, out);
+
+            for (TreeNode child : node.children) {
+                doPrint(child, dimensionCount, depth + 1, out);
+            }
+        }
+
+        private void printCuboid(long cuboidID, int dimensionCount, int depth, PrintWriter out) {
+            StringBuffer sb = new StringBuffer();
+            for (int i = 0; i < depth; i++) {
+                sb.append("    ");
+            }
+            String cuboidName = Cuboid.getDisplayName(cuboidID, dimensionCount);
+            sb.append("|---- Cuboid ").append(cuboidName).append("(" + cuboidID + ")");
+            out.println(sb.toString());
+        }
+
+        private void setRoot(long basicCuboidId) {
+            this.root = new TreeNode(basicCuboidId, 0);
+            this.treeLevels = 0;
+        }
+
+        private void buildIndex() {
+            LinkedList<TreeNode> queue = new LinkedList<>();
+            queue.add(root);
+            while (!queue.isEmpty()) {
+                TreeNode node = queue.removeFirst();
+                index.put(node.cuboidId, node);
+                for (TreeNode child : node.children) {
+                    queue.add(child);
+                }
+            }
+        }
+
+        private void addCuboid(long cuboidId) {
+            TreeNode parent = findBestParent(cuboidId);
+            if (parent != null && parent.cuboidId != cuboidId) {
+                parent.addChild(cuboidId, parent.level);
+                this.treeLevels = Math.max(this.treeLevels, parent.level + 1);
+            }
+        }
+
+        private TreeNode findBestParent(long cuboidId) {
+            TreeNode bestParent = doFindBestParent(cuboidId, root);
+            if (bestParent == null) {
+                throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboidId);
+            }
+            return bestParent;
+        }
+
+        private TreeNode doFindBestParent(long cuboidId, TreeNode parentCuboid) {
+            if (!canDerive(cuboidId, parentCuboid.cuboidId)) {
+                return null;
+            }
+
+            List<TreeNode> candidates = Lists.newArrayList();
+            for (TreeNode childCuboid : parentCuboid.children) {
+                TreeNode candidate = doFindBestParent(cuboidId, childCuboid);
+                if (candidate != null) {
+                    candidates.add(candidate);
+                }
+            }
+            if (candidates.isEmpty()) {
+                candidates.add(parentCuboid);
+            }
+
+            return Collections.min(candidates, new Comparator<TreeNode>() {
+                @Override
+                public int compare(TreeNode o1, TreeNode o2) {
+                    return cuboidComparator.compare(o1.cuboidId, o2.cuboidId);
+                }
+            });
+        }
+
+        private boolean canDerive(long cuboidId, long parentCuboid) {
+            return (cuboidId & ~parentCuboid) == 0;
+        }
+    }
+
+    public static class TreeNode {
+        @JsonProperty("cuboid_id")
+        long cuboidId;
+        @JsonIgnore
+        int level;
+        @JsonProperty("children")
+        List<TreeNode> children = new ArrayList<>();
+
+        public long getCuboidId() {
+            return cuboidId;
+        }
+
+        public int getLevel() {
+            return level;
+        }
+
+        public List<TreeNode> getChildren() {
+            return children;
+        }
+
+        TreeNode(long cuboidId, int level) {
+            this.cuboidId = cuboidId;
+            this.level = level;
+        }
+
+        void addChild(long childId, int parentlevel) {
+            this.children.add(new TreeNode(childId, parentlevel + 1));
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + (int) (cuboidId ^ (cuboidId >>> 32));
+            result = prime * result + level;
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            TreeNode other = (TreeNode) obj;
+            if (cuboidId != other.cuboidId)
+                return false;
+            if (level != other.level)
+                return false;
+            return true;
+        }
+    }
+
+    /**
+     * Compare cuboid according to the cuboid data row count
+     */
+    public static class CuboidCostComparator implements Comparator<Long> {
+        private Map<Long, Long> cuboidStatistics;
+
+        public CuboidCostComparator(Map<Long, Long> cuboidStatistics) {
+            Preconditions.checkArgument(cuboidStatistics != null,
+                    "the input " + cuboidStatistics + " should not be null!!!");
+            this.cuboidStatistics = cuboidStatistics;
+        }
+
+        @Override
+        public int compare(Long cuboid1, Long cuboid2) {
+            Long rowCnt1 = cuboidStatistics.get(cuboid1);
+            Long rowCnt2 = cuboidStatistics.get(cuboid2);
+            if (rowCnt2 == null || rowCnt1 == null) {
+                return Cuboid.cuboidSelectComparator.compare(cuboid1, cuboid2);
+            }
+            return Long.compare(rowCnt1, rowCnt2);
+        }
+    }
+
+    public String getResponsibleKey() {
+        return CubeInstance.class.getName() + "-" + cubeDesc.getName();
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9b67d58c/core-cube/src/test/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerTest.java
new file mode 100644
index 0000000..41fa807
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler.CuboidTree;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class TreeCuboidSchedulerTest {
+
+    @Test
+    public void testCreateCuboidTree() {
+        long basicCuboid = getBaseCuboid(10);
+        List<Long> cuboids = genRandomCuboids(basicCuboid, 200);
+        CuboidTree cuboidTree = CuboidTree.createFromCuboids(cuboids);
+        PrintWriter out = new PrintWriter(System.out);
+        cuboidTree.print(out);
+        out.flush();
+    }
+
+    @Test
+    public void testSpanningChild() {
+        long basicCuboid = getBaseCuboid(10);
+        List<Long> cuboids = genRandomCuboids(basicCuboid, 50);
+        long testCuboid = cuboids.get(10);
+        System.out.println(cuboids);
+        CuboidTree cuboidTree = CuboidTree.createFromCuboids(cuboids);
+        PrintWriter out = new PrintWriter(System.out);
+        cuboidTree.print(out);
+        out.flush();
+
+        List<Long> spanningChildren = cuboidTree.getSpanningCuboid(testCuboid);
+        System.out.println(testCuboid + ":" + spanningChildren);
+    }
+
+    @Test
+    public void testFindBestMatchCuboid() {
+        CuboidTree cuboidTree = createCuboidTree1();
+        PrintWriter out = new PrintWriter(System.out);
+        cuboidTree.print(out);
+        out.flush();
+
+        assertEquals(503L, cuboidTree.findBestMatchCuboid(503L));
+
+        long bestMatch1 = cuboidTree.findBestMatchCuboid(Long.parseLong("100000000", 2));
+        assertEquals(263, bestMatch1);
+
+        long bestMatch2 = cuboidTree.findBestMatchCuboid(Long.parseLong("100010000", 2));
+        assertEquals(304, bestMatch2);
+    }
+
+    private List<Long> genRandomCuboids(long basicCuboidId, int count) {
+        Random random = new Random();
+        List<Long> result = new ArrayList<>();
+        result.add(basicCuboidId);
+        for (int i = 0; i < count; i++) {
+            result.add(random.nextLong() & basicCuboidId);
+        }
+        return result;
+    }
+
+    private long getBaseCuboid(int dimensionCnt) {
+        if (dimensionCnt > 64) {
+            throw new IllegalArgumentException("the dimension count cannot exceed 64");
+        }
+        long result = 0;
+        for (int i = 0; i < dimensionCnt; i++) {
+            result |= (1 << i);
+        }
+        return result;
+    }
+
+    private CuboidTree createCuboidTree1() {
+        List<Long> cuboids = Arrays.asList(504L, 511L, 447L, 383L, 503L, 440L, 496L, 376L, 439L, 487L, 375L, 319L, 432L,
+                480L, 368L, 312L, 423L, 455L, 311L, 359L, 416L, 448L, 304L, 352L, 391L, 295L, 327L, 384L, 288L, 320L,
+                263L);
+        return CuboidTree.createFromCuboids(cuboids,
+                new TreeCuboidScheduler.CuboidCostComparator(simulateStatistics()));
+    }
+
+    private Map<Long, Long> simulateStatistics() {
+        Map<Long, Long> countMap = Maps.newHashMap();
+        countMap.put(511L, 1000000L);
+
+        countMap.put(504L, 900000L);
+        countMap.put(447L, 990000L);
+        countMap.put(383L, 991000L);
+        countMap.put(503L, 980000L);
+
+        countMap.put(440L, 800000L);
+        countMap.put(496L, 890000L);
+        countMap.put(376L, 891000L);
+        countMap.put(439L, 751000L);
+        countMap.put(487L, 751000L);
+        countMap.put(375L, 741000L);
+        countMap.put(319L, 740000L);
+
+        countMap.put(432L, 600000L);
+        countMap.put(480L, 690000L);
+        countMap.put(368L, 691000L);
+        countMap.put(312L, 651000L);
+        countMap.put(423L, 651000L);
+        countMap.put(455L, 541000L);
+        countMap.put(311L, 540000L);
+        countMap.put(359L, 530000L);
+
+        countMap.put(416L, 400000L);
+        countMap.put(448L, 490000L);
+        countMap.put(304L, 491000L);
+        countMap.put(352L, 451000L);
+        countMap.put(391L, 351000L);
+        countMap.put(295L, 141000L);
+        countMap.put(327L, 240000L);
+
+        countMap.put(384L, 100000L);
+        countMap.put(288L, 90000L);
+        countMap.put(320L, 91000L);
+        countMap.put(263L, 51000L);
+
+        return countMap;
+    }
+}


[20/22] kylin git commit: APACHE-KYLIN-2725: add precision for percentile measures of system cubes

Posted by nj...@apache.org.
APACHE-KYLIN-2725: add precision for percentile measures of system cubes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd3e7bf0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd3e7bf0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd3e7bf0

Branch: refs/heads/yaho-cube-planner
Commit: bd3e7bf0cd7e0bda4b55a4cffa2fdb1ffd655864
Parents: 844a6fc
Author: Zhong <nj...@apache.org>
Authored: Thu Sep 7 17:36:58 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:05 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/bd3e7bf0/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
index dad1629..e6914b4 100644
--- a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
+++ b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
@@ -626,7 +626,7 @@ public class CubeDescCreator {
         FunctionDesc function = new FunctionDesc();
         function.setExpression(PercentileMeasureType.FUNC_PERCENTILE);
         function.setParameter(parameterDesc);
-        function.setReturnType(PercentileMeasureType.DATATYPE_PERCENTILE);
+        function.setReturnType("percentile(100)");
 
         MeasureDesc result = new MeasureDesc();
         result.setName(column + "_PERCENTILE");


[03/22] kylin git commit: APACHE-KYLIN-2729: add basic support classes for cube planner algorithms

Posted by nj...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/test/resources/statistics.txt
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/statistics.txt b/core-cube/src/test/resources/statistics.txt
new file mode 100644
index 0000000..35f69fc
--- /dev/null
+++ b/core-cube/src/test/resources/statistics.txt
@@ -0,0 +1,4092 @@
+1 5408
+2 12
+3 59114
+4 5393
+5 13545958
+6 59352
+7 48900622
+8 234
+9 622632
+10 2513
+11 2684805
+12 661703
+13 51457071
+14 3003220
+15 87905334
+16 4
+17 20057
+18 48
+19 185091
+20 21381
+21 27129912
+22 196464
+23 67684891
+24 905
+25 1409380
+26 8032
+27 5367438
+28 1540851
+29 66914540
+30 5780199
+31 102382118
+32 5
+33 26824
+34 60
+35 250433
+36 26763
+37 30256052
+38 261493
+39 69394709
+40 387
+41 665893
+42 2849
+43 2954813
+44 707431
+45 61390587
+46 3240484
+47 97487859
+48 16
+49 78962
+50 189
+51 648097
+52 80270
+53 46703642
+54 675226
+55 85512868
+56 1138
+57 1463428
+58 8619
+59 5848317
+60 1577917
+61 77312551
+62 6472745
+63 111122050
+64 14
+65 42855
+66 144
+67 337000
+68 44890
+69 33660047
+70 361045
+71 73187752
+72 427
+73 715905
+74 3192
+75 3277233
+76 765421
+77 66688961
+78 3649848
+79 100898382
+80 40
+81 116673
+82 404
+83 846531
+84 123061
+85 51207630
+86 908398
+87 89543966
+89 1645138
+90 9255
+91 6589067
+92 1780584
+93 83045129
+94 7349741
+95 113014051
+96 54
+97 170610
+98 585
+99 1112616
+100 182519
+101 46860213
+102 1270670
+103 83382182
+104 596
+105 823678
+106 3734
+107 3900149
+108 884212
+109 74220549
+110 4350663
+111 106240989
+112 149
+113 407851
+114 1495
+115 2349130
+116 436830
+117 63626889
+118 2661185
+119 98988611
+120 1491
+121 1871944
+122 10494
+123 7619644
+124 2048333
+125 89765741
+126 8703141
+127 117905202
+128 6
+129 31335
+130 72
+131 278154
+132 32528
+133 33299431
+134 289515
+135 72241688
+136 628
+137 671628
+138 3798
+139 2945467
+140 728973
+141 64709349
+142 3205057
+143 99730017
+144 19
+145 86296
+146 217
+147 704603
+148 87692
+149 50352721
+150 728318
+151 89072848
+152 1169
+153 1468982
+154 8697
+155 5821059
+156 1602799
+157 82438973
+158 6402424
+159 113177011
+160 22
+161 111200
+162 263
+163 821474
+164 113822
+165 41785539
+166 872073
+167 76702603
+168 734
+169 753231
+170 4143
+171 3512464
+172 809672
+173 71693751
+174 3856651
+175 104344446
+176 67
+177 273672
+178 744
+179 1742709
+180 285398
+181 57385824
+182 1920352
+183 91758819
+184 1332
+185 1689133
+186 9413
+187 6897407
+188 1821788
+189 88978769
+190 7611629
+191 116410794
+192 51
+193 171884
+194 560
+195 1169383
+196 183414
+197 49288586
+198 1305325
+199 85851630
+200 815
+201 842521
+202 4624
+203 4010811
+204 907399
+205 77349281
+206 4354569
+207 109042390
+208 144
+209 415680
+210 1523
+211 2475079
+212 441787
+213 66734849
+214 2864972
+215 100470889
+216 1505
+217 1871044
+218 10528
+219 7754601
+220 2042714
+221 93029295
+222 8834572
+223 117923381
+224 160
+225 454208
+226 1684
+227 2583569
+228 503525
+229 57311428
+230 2989675
+231 90863923
+232 1048
+233 1140000
+234 6050
+235 5382389
+236 1238979
+237 84241270
+238 6191583
+239 114362710
+240 428
+241 972308
+242 4150
+243 4729682
+244 1089252
+245 72065430
+246 5730712
+247 104402615
+248 1939
+249 2462027
+250 13624
+251 10071894
+252 2712730
+253 97601168
+254 11673557
+255 122190940
+256 2
+257 9498
+258 24
+259 79945
+260 10752
+261 14258615
+262 107239
+263 50856457
+264 452
+265 697726
+266 4490
+267 2822825
+268 932822
+269 53639060
+270 3662872
+271 90489596
+272 8
+273 29798
+274 95
+275 223957
+276 41341
+277 28442950
+278 332878
+279 68930124
+280 1699
+281 1531510
+282 13519
+283 5635662
+284 1989662
+285 69678635
+286 6989874
+287 103854924
+288 10
+289 38592
+290 120
+291 295819
+292 51637
+293 31369690
+294 434324
+295 70938987
+296 671
+297 721499
+298 5038
+299 3147896
+300 968177
+301 64784337
+302 4035226
+303 98482649
+304 32
+305 101297
+306 365
+307 712465
+308 143822
+309 48962693
+310 1050839
+311 88025320
+312 2013
+313 1585225
+314 14261
+315 6129197
+316 2060910
+317 81099274
+318 7634802
+319 111395308
+320 27
+321 57497
+322 257
+323 380724
+324 79504
+325 34879820
+326 587405
+327 76210771
+328 734
+329 789662
+330 5397
+331 3524986
+332 1088152
+333 68215167
+334 4691841
+335 104893830
+336 75
+337 143289
+338 703
+339 925065
+340 214788
+341 53155396
+342 1384457
+343 93315403
+344 2163
+345 1777001
+346 15550
+347 6864447
+348 2381043
+349 85018646
+350 8861967
+351 114587784
+352 103
+353 203441
+354 1031
+355 1216511
+356 296217
+357 48144102
+358 1789235
+359 86052906
+360 965
+361 908466
+362 6345
+363 4128921
+364 1268319
+365 76413089
+366 5553302
+367 108727635
+368 273
+369 459035
+370 2568
+371 2479945
+372 676543
+373 65867786
+374 3570437
+375 101630311
+376 2507
+377 1987608
+378 17331
+379 7980360
+380 2679601
+381 92363718
+382 10275128
+383 117942645
+384 12
+385 45711
+386 143
+387 332427
+388 63139
+389 34188123
+390 513289
+391 74310480
+392 1060
+393 744405
+394 6322
+395 3144421
+396 998196
+397 67000938
+398 4092455
+399 102479781
+400 37
+401 112496
+402 405
+403 801759
+404 159812
+405 51596478
+406 1182427
+407 90983246
+408 2029
+409 1581472
+410 14290
+411 6061271
+412 2047019
+413 84461228
+414 7742372
+415 112474685
+416 43
+417 136352
+418 496
+419 902897
+420 198401
+421 43850605
+422 1324526
+423 78791391
+424 1186
+425 829942
+426 6923
+427 3727578
+428 1138486
+429 73609409
+430 4872398
+431 107810620
+432 126
+433 314911
+434 1339
+435 1863528
+436 460075
+437 59860345
+438 2628087
+439 95065035
+440 2255
+441 1797843
+442 15594
+443 7234754
+444 2398192
+445 90382853
+446 9184313
+447 118024610
+448 100
+449 204678
+450 1013
+451 1288671
+452 311368
+453 51461477
+454 1940945
+455 88461722
+456 1319
+457 913332
+458 7819
+459 4144085
+460 1279201
+461 79110460
+462 5635626
+463 110369460
+464 273
+465 477392
+466 2657
+467 2622297
+468 712921
+469 69125337
+470 3764291
+471 101922838
+472 2531
+473 2001102
+474 17156
+475 8011737
+476 2725091
+477 95464996
+478 10557559
+479 121667004
+480 302
+481 514289
+482 2955
+483 2727750
+484 764150
+485 57613471
+486 3864725
+487 91514836
+488 1671
+489 1216475
+490 10000
+491 5622217
+492 1711059
+493 87066187
+494 7699073
+495 115145694
+496 770
+497 1059840
+498 7002
+499 4971895
+500 1543099
+501 73147277
+502 6992897
+503 105256346
+504 3194
+505 2628403
+506 22318
+507 10634614
+508 3581598
+509 99591089
+510 13694936
+511 123805801
+512 97
+513 438583
+514 1082
+515 2709770
+516 467793
+517 62233795
+518 3236365
+519 92864138
+520 873
+521 1201180
+522 4916
+523 5992226
+524 1295951
+525 89881852
+526 6821855
+527 116111968
+528 290
+529 1027455
+530 3181
+531 5372086
+532 1118502
+533 76874973
+534 6477401
+535 106174660
+536 1934
+537 2683421
+538 13540
+539 11305324
+540 2991570
+541 103334266
+542 13295977
+543 124348132
+544 270
+545 1071956
+546 3026
+547 5786650
+548 1181191
+549 67812072
+550 7082943
+551 98196271
+552 1181
+553 1726665
+554 7259
+555 8291732
+556 1877832
+557 93139506
+558 10029949
+559 119823816
+560 789
+561 2302526
+562 8328
+563 9912121
+564 2601558
+565 82274208
+566 12455751
+567 110555733
+568 2644
+569 3732049
+570 19144
+571 15286252
+572 4134722
+573 106709277
+574 18281589
+575 128436771
+576 528
+577 1488216
+578 5508
+579 7357343
+580 1692952
+581 70505609
+582 9163234
+583 99934902
+584 1509
+585 2198656
+586 9779
+587 10127264
+588 2482129
+589 97628745
+590 12274594
+591 122862474
+592 1472
+593 3124699
+594 14405
+595 12289082
+596 3626702
+597 83834130
+598 15663342
+599 112128796
+600 3392
+601 4553823
+602 24896
+603 17313176
+604 5234246
+605 110223356
+606 21393439
+607 129849677
+608 1706
+609 2983727
+610 15499
+611 12215091
+612 3493957
+613 74709369
+614 15524952
+615 102572764
+616 2894
+617 3633552
+618 20698
+619 14875112
+620 4164119
+621 99922065
+622 18698435
+623 125227448
+624 4256
+625 5549352
+626 36184
+627 18803250
+628 6673860
+629 87286090
+630 24874079
+631 115575487
+632 6459
+633 7016394
+634 47556
+635 23976463
+636 8150783
+637 112752491
+638 30924190
+639 133441422
+640 266
+641 1207316
+642 3052
+643 6636130
+644 1300828
+645 69384103
+646 8156139
+647 98959683
+648 1398
+649 1923357
+650 8325
+651 9397687
+652 2082573
+653 95659787
+654 11220516
+655 119557748
+656 796
+657 2638421
+658 8734
+659 11505399
+660 2920808
+661 82860830
+662 14608740
+663 110360653
+664 2664
+665 4175381
+666 19534
+667 16801270
+668 4630930
+669 108806369
+670 20353452
+671 130528080
+672 939
+673 2827946
+674 10260
+675 12029336
+676 3182512
+677 73512155
+678 15772458
+679 102756341
+680 2344
+681 3476032
+682 16025
+683 14821628
+684 3912061
+685 100378233
+686 18580297
+687 124831025
+688 2621
+689 5304217
+690 25764
+691 18836482
+692 6247823
+693 86432297
+694 24530141
+695 115054234
+696 4916
+697 6820121
+698 37208
+699 24204404
+700 7854361
+701 110867598
+702 30209819
+703 135122998
+704 1604
+705 3317097
+706 15294
+707 13997817
+708 3885042
+709 75085038
+710 17901482
+711 103964100
+712 3112
+713 3999443
+714 21348
+715 16609940
+716 4647775
+717 100722007
+718 21008944
+719 127918093
+720 4244
+721 6276421
+722 36834
+723 21104546
+724 7602232
+725 88383830
+726 27954151
+727 116964820
+728 6670
+729 7958519
+730 48705
+731 26741535
+732 9310460
+733 114567332
+734 34012957
+735 137067139
+736 5334
+737 6688120
+738 45994
+739 21913788
+740 8199538
+741 78744048
+742 28615773
+743 108064945
+744 7301
+745 7374315
+746 51827
+747 24569184
+748 8975876
+749 104718633
+750 31172497
+751 131132532
+752 12668
+753 11226730
+754 98340
+755 30834996
+756 14086985
+757 93251462
+758 40453981
+759 120846110
+760 15512
+761 12738469
+762 110887
+763 36298409
+764 15874117
+765 115913270
+766 45644274
+767 138038027
+768 192
+769 494272
+770 2101
+771 2924324
+772 764314
+773 64313489
+774 4326195
+775 97104611
+776 1311
+777 1305415
+778 8457
+779 6165294
+780 1901623
+781 90611558
+782 8669189
+783 118554901
+784 577
+785 1147131
+786 5889
+787 5536938
+788 1721834
+789 78132034
+790 8046126
+791 108884068
+792 3229
+793 2866087
+794 22554
+795 11657118
+796 4013420
+797 105254429
+798 15495349
+799 127470105
+800 538
+801 1181321
+802 5741
+803 6020531
+804 1803671
+805 69690385
+806 8449098
+807 98671282
+808 1814
+809 1877892
+810 12291
+811 8754810
+812 2759541
+813 97772121
+814 12082368
+815 120788386
+816 1528
+817 2450462
+818 15039
+819 10372968
+820 3544857
+821 83722940
+822 14334373
+823 110816273
+824 4330
+825 3972188
+826 31649
+827 15765886
+828 5545733
+829 109234519
+830 20713321
+831 130127499
+832 1002
+833 1577814
+834 9545
+835 7576178
+836 2439206
+837 71766071
+838 10915295
+839 100458148
+840 2343
+841 2328703
+842 16180
+843 10483353
+844 3441733
+845 99813101
+846 14611780
+847 121316162
+848 2715
+849 3241664
+850 23634
+851 12808809
+852 4752410
+853 85834475
+854 17805351
+855 113391766
+856 5600
+857 4757337
+858 40361
+859 18190693
+860 6739328
+861 112781190
+862 24058555
+863 133824858
+864 2874
+865 3159084
+866 24521
+867 12492065
+868 4494122
+869 76201036
+870 17726844
+871 102131562
+872 4475
+873 3832627
+874 32005
+875 15438631
+876 5410669
+877 103092393
+878 21025747
+879 124629007
+880 7014
+881 5815341
+882 54957
+883 19411364
+884 8002622
+885 88589435
+886 26878537
+887 118293969
+888 10339
+889 7377283
+890 72321
+891 24837882
+892 10041432
+893 114001415
+894 33362816
+895 134556926
+896 528
+897 1302023
+898 5874
+899 6916181
+900 2002364
+901 70992688
+902 9738517
+903 99824048
+904 2204
+905 2060794
+906 13912
+907 9725899
+908 2976365
+909 97543583
+910 13193140
+911 121717746
+912 1564
+913 2787402
+914 15871
+915 12101312
+916 3938817
+917 85048610
+918 16807580
+919 113186566
+920 4445
+921 4309356
+922 32492
+923 17342856
+924 5976510
+925 110425493
+926 23151915
+927 129144702
+928 1707
+929 2935631
+930 17601
+931 12430272
+932 4154926
+933 73795002
+934 17277433
+935 104447950
+936 3748
+937 3655594
+938 26258
+939 15269699
+940 5086842
+941 102039383
+942 20774859
+943 125357623
+944 4693
+945 5489921
+946 42290
+947 19427124
+948 7514459
+949 87672236
+950 26451367
+951 115013581
+952 8006
+953 7124350
+954 60434
+955 24593613
+956 9513495
+957 114569527
+958 33288566
+959 134472524
+960 2900
+961 3466345
+962 25122
+963 14315270
+964 4969555
+965 77208126
+966 19818490
+967 105788076
+968 4990
+969 4221110
+970 33871
+971 17255470
+972 5969058
+973 102263401
+974 23737233
+975 128807532
+976 7382
+977 6537423
+978 58610
+979 21948136
+980 8919704
+981 90888626
+982 30390361
+983 119051840
+984 10928
+985 8203126
+986 76521
+987 27633249
+988 11140419
+989 114440012
+990 36992653
+991 135276469
+992 8907
+993 6923754
+994 70349
+995 22454011
+996 9467754
+997 80151964
+998 31126922
+999 110226875
+1000 11535
+1001 7690684
+1002 80056
+1003 25245134
+1004 10480355
+1005 107198218
+1006 34288000
+1007 129185160
+1008 20582
+1009 11741125
+1010 146083
+1011 32064284
+1012 15900178
+1013 93412095
+1014 42446327
+1015 122663267
+1016 24255
+1017 13339271
+1018 166816
+1019 36578306
+1020 18058472
+1021 118691906
+1022 49121357
+1023 140113432
+1024 31
+1025 150413
+1026 366
+1027 1158644
+1028 157542
+1029 51738914
+1030 1251433
+1031 87564031
+1032 872
+1033 842251
+1034 4476
+1035 4113036
+1036 900240
+1037 78408308
+1038 4567962
+1039 111542416
+1040 93
+1041 402568
+1042 1086
+1043 2554935
+1044 431312
+1045 67925585
+1046 2942411
+1047 102916480
+1048 1527
+1049 1896622
+1050 10210
+1051 8192965
+1052 2063099
+1053 93601052
+1054 9265168
+1055 121858401
+1056 88
+1057 370225
+1058 1025
+1059 2417970
+1060 396557
+1061 61259477
+1062 2777133
+1063 93660336
+1064 1045
+1065 1001241
+1066 5311
+1067 5111023
+1068 1082547
+1069 85511233
+1070 5672325
+1071 116016162
+1072 265
+1073 895954
+1074 2848
+1075 4821805
+1076 995465
+1077 75619690
+1078 5751715
+1079 107422928
+1080 1819
+1081 2306325
+1082 11993
+1083 9782219
+1084 2516472
+1085 100548867
+1086 11462175
+1087 125532977
+1088 192
+1089 629408
+1090 1986
+1091 3545722
+1092 706626
+1093 63252380
+1094 4511461
+1095 95363459
+1096 1256
+1097 1406524
+1098 7031
+1099 6598640
+1100 1581968
+1101 89406248
+1102 7921177
+1103 117127451
+1104 546
+1105 1500980
+1106 5318
+1107 6740546
+1108 1748642
+1109 77488092
+1110 8648849
+1111 107904509
+1112 2351
+1113 3158581
+1114 16590
+1115 12308887
+1116 3591894
+1117 103824241
+1118 15157568
+1119 124572802
+1120 528
+1121 1293488
+1122 5225
+1123 6207972
+1124 1503400
+1125 68975282
+1126 7991050
+1127 97278780
+1128 1646
+1129 1936460
+1130 10057
+1131 8990579
+1132 2211374
+1133 93764350
+1134 10894996
+1135 118538464
+1136 1422
+1137 2711838
+1138 13151
+1139 10697510
+1140 3240677
+1141 82089910
+1142 13911964
+1143 112665340
+1144 3216
+1145 4105046
+1146 23032
+1147 15785201
+1148 4793386
+1149 106967458
+1150 19875798
+1151 128446296
+1152 40
+1153 192297
+1154 475
+1155 1396806
+1156 202836
+1157 52637774
+1158 1606679
+1159 89369726
+1160 902
+1161 828488
+1162 4563
+1163 4057633
+1164 892115
+1165 78947682
+1166 4628052
+1167 111422777
+1168 120
+1169 516183
+1170 1391
+1171 3097751
+1172 564017
+1173 69601145
+1174 3637734
+1175 102905271
+1176 1546
+1177 1933969
+1178 10305
+1179 8269340
+1180 2104002
+1181 95781245
+1182 9426626
+1183 120561698
+1184 89
+1185 369612
+1186 1026
+1187 2408796
+1188 397318
+1189 59904945
+1190 2820020
+1191 94660030
+1192 1045
+1193 999782
+1194 5330
+1195 5094626
+1196 1093516
+1197 86313767
+1198 5690937
+1199 114097583
+1200 267
+1201 891221
+1202 2816
+1203 4842290
+1204 987372
+1205 75279605
+1206 5732936
+1207 105857115
+1208 1809
+1209 2307523
+1210 12149
+1211 9964552
+1212 2526404
+1213 100760890
+1214 11414974
+1215 123666639
+1216 274
+1217 755065
+1218 2729
+1219 3977942
+1220 870519
+1221 64659605
+1222 5120758
+1223 96933647
+1224 1278
+1225 1396320
+1226 7119
+1227 6565673
+1228 1557192
+1229 90355168
+1230 7938956
+1231 115782522
+1232 774
+1233 1720285
+1234 7329
+1235 7194896
+1236 2048618
+1237 78967236
+1238 9406784
+1239 107980362
+1240 2353
+1241 3123420
+1242 16670
+1243 12183951
+1244 3583355
+1245 103563328
+1246 15079836
+1247 124974132
+1248 525
+1249 1315885
+1250 5228
+1251 6242417
+1252 1522222
+1253 68808113
+1254 8105522
+1255 97128744
+1256 1671
+1257 1934414
+1258 10089
+1259 8992037
+1260 2195395
+1261 92646810
+1262 10827707
+1263 120784730
+1264 1418
+1265 2700373
+1266 13078
+1267 10786324
+1268 3231141
+1269 83046435
+1270 13975930
+1271 111979610
+1272 3220
+1273 4102509
+1274 23118
+1275 15607046
+1276 4771277
+1277 106799854
+1278 19520816
+1279 128796256
+1280 62
+1281 184096
+1282 708
+1283 1226094
+1284 286421
+1285 53802398
+1286 1996014
+1287 90620362
+1288 1357
+1289 924841
+1290 7488
+1291 4331679
+1292 1321421
+1293 81221055
+1294 6008660
+1295 111894723
+1296 186
+1297 458807
+1298 2043
+1299 2766075
+1300 720141
+1301 71253190
+1302 4155892
+1303 103750216
+1304 2519
+1305 2084785
+1306 17022
+1307 8646435
+1308 2851706
+1309 98202027
+1310 11287111
+1311 122619487
+1312 176
+1313 420756
+1314 1880
+1315 2561851
+1316 659248
+1317 61062843
+1318 3869343
+1319 95103270
+1320 1599
+1321 1108260
+1322 8792
+1323 5326849
+1324 1588410
+1325 87430404
+1326 7470599
+1327 115919794
+1328 505
+1329 981712
+1330 5058
+1331 5136174
+1332 1520407
+1333 78164384
+1334 7319615
+1335 107199735
+1336 2967
+1337 2463767
+1338 20121
+1339 10261048
+1340 3464105
+1341 100962295
+1342 13606407
+1343 122701080
+1344 364
+1345 701750
+1346 3503
+1347 3800034
+1348 1137022
+1349 64795529
+1350 5848348
+1351 96736498
+1352 1994
+1353 1526157
+1354 11745
+1355 6964018
+1356 2305917
+1357 91172102
+1358 9880389
+1359 119077644
+1360 1013
+1361 1622751
+1362 9579
+1363 7019881
+1364 2523931
+1365 80951123
+1366 10428309
+1367 109408699
+1368 3941
+1369 3277940
+1370 27831
+1371 12852475
+1372 4758310
+1373 106022332
+1374 17495641
+1375 126719249
+1376 972
+1377 1399891
+1378 8883
+1379 6522581
+1380 2149418
+1381 69594418
+1382 9537448
+1383 100195532
+1384 2646
+1385 2075924
+1386 16513
+1387 9174104
+1388 3073036
+1389 96008133
+1390 13150409
+1391 120614695
+1392 2544
+1393 2847171
+1394 21947
+1395 11133816
+1396 4270758
+1397 84317788
+1398 15877777
+1399 112157213
+1400 5329
+1401 4394623
+1402 37521
+1403 16359478
+1404 6221391
+1405 108191440
+1406 22252708
+1407 129282760
+1408 80
+1409 233598
+1410 919
+1411 1529897
+1412 372629
+1413 54941648
+1414 2403095
+1415 93082547
+1416 1405
+1417 931537
+1418 7578
+1419 4290188
+1420 1319386
+1421 81636241
+1422 6049584
+1423 111209518
+1424 239
+1425 588606
+1426 2633
+1427 3298108
+1428 944159
+1429 72146846
+1430 4961109
+1431 105176126
+1432 2542
+1433 2085325
+1434 16845
+1435 8528145
+1436 2872494
+1437 97299053
+1438 11408785
+1439 122170761
+1440 175
+1441 428398
+1442 1857
+1443 2577017
+1444 672969
+1445 62750644
+1446 3850272
+1447 93736662
+1448 1602
+1449 1130498
+1450 8813
+1451 5381949
+1452 1619208
+1453 89143817
+1454 7451329
+1455 115301172
+1456 503
+1457 997074
+1458 5019
+1459 5067096
+1460 1515936
+1461 77314474
+1462 7324790
+1463 107872289
+1464 2949
+1465 2504573
+1466 19777
+1467 10264209
+1468 3491214
+1469 101733794
+1470 13764677
+1471 126159667
+1472 519
+1473 828924
+1474 4847
+1475 4166168
+1476 1340404
+1477 66334297
+1478 6439460
+1479 96008951
+1480 2005
+1481 1507150
+1482 11846
+1483 6968586
+1484 2282402
+1485 91127119
+1486 9843793
+1487 118194925
+1488 1424
+1489 1831180
+1490 12770
+1491 7605776
+1492 2883433
+1493 80125968
+1494 11202391
+1495 109486787
+1496 3918
+1497 3307601
+1498 27919
+1499 12955575
+1500 4738709
+1501 103773671
+1502 17275804
+1503 127329438
+1504 969
+1505 1422291
+1506 8909
+1507 6507932
+1508 2178852
+1509 70387439
+1510 9624012
+1511 101004435
+1512 2633
+1513 2073074
+1514 16450
+1515 9257057
+1516 3095246
+1517 96744708
+1518 13164609
+1519 120792891
+1520 2537
+1521 2849173
+1522 21678
+1523 11126753
+1524 4214940
+1525 83496541
+1526 16227929
+1527 111759246
+1528 5282
+1529 4327984
+1530 36901
+1531 16352366
+1532 6228279
+1533 109868812
+1534 22306845
+1535 128783268
+1536 613
+1537 2730648
+1538 7248
+1539 13278299
+1540 3049614
+1541 79907265
+1542 17488876
+1543 107021427
+1544 2039
+1545 3426089
+1546 13111
+1547 15894701
+1548 3779090
+1549 103817282
+1550 20610011
+1551 127619132
+1552 1850
+1553 5750197
+1554 20084
+1555 21326501
+1556 6829385
+1557 90140817
+1558 29179696
+1559 119891574
+1560 4089
+1561 7286104
+1562 31998
+1563 26759490
+1564 8364412
+1565 116915755
+1566 34853541
+1567 135295601
+1568 1947
+1569 5418503
+1570 20958
+1571 20571143
+1572 6344275
+1573 81816479
+1574 27592777
+1575 108579191
+1576 3733
+1577 6083457
+1578 28179
+1579 23099311
+1580 7149218
+1581 108603139
+1582 30825252
+1583 131097109
+1584 5443
+1585 10100080
+1586 53275
+1587 30137763
+1588 12414498
+1589 94443293
+1590 40499460
+1591 122045858
+1592 8237
+1593 11475670
+1594 66003
+1595 35381710
+1596 13907552
+1597 119123389
+1598 47110549
+1599 139272771
+1600 4061
+1601 7127017
+1602 36882
+1603 24401845
+1604 8922688
+1605 84138449
+1606 33420425
+1607 112205558
+1608 5886
+1609 7772691
+1610 44202
+1611 27042921
+1612 9659516
+1613 108984786
+1614 36408983
+1615 130079215
+1616 10558
+1617 12691029
+1618 91335
+1619 34920030
+1620 16478557
+1621 96095309
+1622 46744861
+1623 123508848
+1624 13624
+1625 14030341
+1626 102580
+1627 39459290
+1628 17864280
+1629 121671572
+1630 53134291
+1631 140735841
+1632 11493
+1633 12014204
+1634 95520
+1635 33843551
+1636 15393515
+1637 86890246
+1638 44868182
+1639 114698301
+1640 14164
+1641 12661312
+1642 102269
+1643 37646637
+1644 16290134
+1645 112225670
+1646 49119976
+1647 135335977
+1648 27272
+1649 19243478
+1650 206764
+1651 45579973
+1652 25651324
+1653 98142216
+1654 58063137
+1655 126425142
+1656 31033
+1657 21408777
+1658 224421
+1659 49646834
+1660 27595106
+1661 122883998
+1662 63969490
+1663 144683259
+1664 617
+1665 2679240
+1666 7142
+1667 13283653
+1668 2952028
+1669 80788327
+1670 17722175
+1671 107927325
+1672 2063
+1673 3445371
+1674 13272
+1675 16066069
+1676 3812631
+1677 103282457
+1678 20565602
+1679 125650673
+1680 1844
+1681 5774665
+1682 20286
+1683 21770311
+1684 6799693
+1685 91092592
+1686 28744229
+1687 119392777
+1688 4083
+1689 7211377
+1690 32261
+1691 26442435
+1692 8343834
+1693 115077456
+1694 35234902
+1695 136484876
+1696 1945
+1697 5426386
+1698 21077
+1699 20802936
+1700 6367298
+1701 82046460
+1702 27747469
+1703 110219853
+1704 3779
+1705 6155539
+1706 28029
+1707 23546836
+1708 7168340
+1709 106466176
+1710 30803094
+1711 131112257
+1712 5341
+1713 10034176
+1714 53170
+1715 30473019
+1716 12384347
+1717 94382900
+1718 40983184
+1719 123154627
+1720 8178
+1721 11463619
+1722 66948
+1723 35258213
+1724 13885331
+1725 115631945
+1726 46896987
+1727 138729236
+1728 4027
+1729 7036632
+1730 36665
+1731 24385441
+1732 8812806
+1733 83382945
+1734 33703350
+1735 111467501
+1736 5937
+1737 7981053
+1738 43270
+1739 27665655
+1740 9770689
+1741 108871177
+1742 35993072
+1743 133627510
+1744 10501
+1745 12708057
+1746 91780
+1747 34599617
+1748 16286618
+1749 94619754
+1750 46654440
+1751 124234980
+1752 13604
+1753 14157541
+1754 104051
+1755 40103516
+1756 17871019
+1757 119564746
+1758 51707472
+1759 141461884
+1760 11643
+1761 12067915
+1762 96253
+1763 34014423
+1764 15465732
+1765 86580327
+1766 45226540
+1767 116041041
+1768 13865
+1769 12746628
+1770 103219
+1771 36241695
+1772 16213272
+1773 111309833
+1774 48414814
+1775 135449250
+1776 27000
+1777 19411950
+1778 205500
+1779 44434714
+1780 25714325
+1781 99059044
+1782 57678269
+1783 126866168
+1784 30973
+1785 21102620
+1786 222239
+1787 50330284
+1788 27371928
+1789 122497188
+1790 64093612
+1791 143298155
+1792 1240
+1793 2924331
+1794 13496
+1795 13631199
+1796 4468294
+1797 80097158
+1798 20106155
+1799 108928714
+1800 3282
+1801 3649010
+1802 22380
+1803 16781603
+1804 5388194
+1805 106628403
+1806 23564226
+1807 128033285
+1808 3618
+1809 6021472
+1810 37198
+1811 22156554
+1812 8609761
+1813 93378196
+1814 31350349
+1815 121356005
+1816 6988
+1817 7594121
+1818 54875
+1819 27270994
+1820 10590865
+1821 117257800
+1822 37828233
+1823 136596579
+1824 3646
+1825 5656922
+1826 36404
+1827 21482255
+1828 7994317
+1829 83454123
+1830 30351318
+1831 109286465
+1832 6270
+1833 6425176
+1834 46201
+1835 23962566
+1836 8980191
+1837 107620761
+1838 33626199
+1839 132733059
+1840 9921
+1841 10458528
+1842 88975
+1843 30993855
+1844 14208475
+1845 95250681
+1846 42769354
+1847 123650134
+1848 13975
+1849 11857235
+1850 106420
+1851 36574084
+1852 16257651
+1853 118851270
+1854 50121180
+1855 140997664
+1856 7158
+1857 7372328
+1858 61335
+1859 25408870
+1860 10522563
+1861 84490786
+1862 35395854
+1863 113763173
+1864 9770
+1865 8320083
+1866 70997
+1867 28165783
+1868 11750214
+1869 109864335
+1870 39198261
+1871 132867109
+1872 18628
+1873 13071342
+1874 147439
+1875 35759298
+1876 18399007
+1877 96994118
+1878 49414422
+1879 125529665
+1880 22337
+1881 14802666
+1882 165092
+1883 41444909
+1884 20532132
+1885 121596514
+1886 54883602
+1887 141485763
+1889 12520099
+1890 145714
+1891 35424339
+1892 17663868
+1893 88137185
+1894 47167541
+1895 116264847
+1896 22685
+1897 13224942
+1898 156980
+1899 37818236
+1900 18457151
+1901 113678427
+1902 51664666
+1903 134162002
+1904 44355
+1905 19805951
+1906 310302
+1907 46353216
+1908 28026476
+1909 100495308
+1910 59486731
+1911 127948851
+1912 49058
+1913 21725542
+1914 329641
+1915 51218460
+1916 30049289
+1917 124537971
+1918 66983942
+1919 144770570
+1920 1227
+1921 2928344
+1922 13437
+1923 13616235
+1924 4444041
+1925 80096207
+1926 19749563
+1927 107817260
+1928 3278
+1929 3626551
+1930 22244
+1931 16535803
+1932 5412984
+1933 106282543
+1934 23433780
+1935 126540705
+1936 3683
+1937 6086630
+1938 37432
+1939 22284984
+1940 8580463
+1941 93622551
+1942 31617410
+1943 121838198
+1944 7043
+1945 7673323
+1946 54983
+1947 27831814
+1948 10704915
+1949 117534939
+1950 37519581
+1951 138812008
+1952 3659
+1953 5702609
+1954 36759
+1955 21267091
+1956 8048006
+1957 83274001
+1958 30193501
+1959 110759141
+1960 6207
+1961 6453541
+1962 47072
+1963 24071255
+1964 8976310
+1965 108266609
+1966 33115615
+1967 132031002
+1968 9967
+1969 10220194
+1970 88636
+1971 31213580
+1972 14434863
+1973 96789994
+1974 42471936
+1975 123722116
+1976 14046
+1977 11895366
+1978 106772
+1979 36685350
+1980 16121825
+1981 118751144
+1982 49926109
+1983 140689183
+1984 7189
+1985 7527179
+1986 61452
+1987 25329724
+1988 10616607
+1989 84677617
+1990 36118985
+1991 112532106
+1992 9835
+1993 8251387
+1994 71885
+1995 28035767
+1996 11715520
+1997 110480934
+1998 38941603
+1999 131575707
+2000 18790
+2001 13144752
+2002 146177
+2003 36022195
+2004 18497284
+2005 97131033
+2006 48509887
+2007 125631086
+2008 22669
+2009 14660464
+2010 164199
+2011 40959811
+2012 20388574
+2013 121600974
+2014 55479463
+2015 143099953
+2016 19157
+2017 12698361
+2018 148484
+2019 34577209
+2020 17790461
+2021 87213355
+2022 47476392
+2023 115442622
+2024 22629
+2025 13179261
+2026 154089
+2027 37538204
+2028 18377556
+2029 114492629
+2030 50854718
+2031 136212446
+2032 44950
+2033 20224756
+2034 313127
+2035 46192224
+2036 28234029
+2037 99809422
+2038 60251012
+2039 128322426
+2040 48643
+2041 21659921
+2042 333423
+2043 51327747
+2044 29698685
+2045 125287720
+2046 65541052
+2047 143363712
+2048 7442
+2049 16865576
+2050 86014
+2051 58493528
+2052 11942184
+2053 89632581
+2054 33477680
+2055 114775367
+2056 963389
+2057 62981401
+2058 4060470
+2059 106535613
+2060 36534798
+2061 117901650
+2062 61611213
+2063 136609704
+2064 28157
+2065 32701475
+2066 266899
+2067 80002045
+2068 20605052
+2069 103283230
+2070 44954693
+2071 126825626
+2072 2250088
+2073 83828002
+2074 8204614
+2075 122431797
+2076 48724593
+2077 127569021
+2078 74101651
+2079 147219073
+2080 36778
+2081 36023749
+2082 365157
+2083 80885359
+2084 22836215
+2085 102704007
+2086 46195873
+2087 124924944
+2088 1007977
+2089 73640279
+2090 4400657
+2091 116541566
+2092 42983632
+2093 123015751
+2094 68629830
+2095 141542925
+2096 107596
+2097 55596098
+2098 928674
+2099 100144751
+2100 32428219
+2101 114069321
+2102 56315717
+2103 132896191
+2104 2301083
+2105 95488907
+2106 8992628
+2107 129755866
+2108 55800262
+2109 133664018
+2110 81270095
+2111 148867980
+2112 60818
+2113 39478388
+2114 496621
+2115 87346550
+2116 24864797
+2117 104142521
+2118 49324042
+2119 127890693
+2120 1085707
+2121 80954470
+2122 5077239
+2123 120109435
+2124 46621404
+2125 125315012
+2126 72216620
+2127 144409677
+2128 172981
+2129 58845074
+2130 1246896
+2131 104330868
+2132 35009260
+2133 116873522
+2134 59436855
+2135 135497858
+2136 2545438
+2137 100546572
+2138 10140144
+2139 135715986
+2140 59536760
+2141 135964268
+2142 85499041
+2143 150029514
+2144 238274
+2145 54642555
+2146 1714172
+2147 98350715
+2148 33013034
+2149 113006861
+2150 56919070
+2151 131203018
+2152 1251207
+2153 89232062
+2154 5957049
+2155 126359432
+2156 51413519
+2157 129235200
+2158 76044270
+2159 144135646
+2160 582156
+2161 75540121
+2162 3578468
+2163 115717529
+2164 43538840
+2165 122547316
+2166 66733129
+2167 138354560
+2168 2911859
+2169 108527925
+2170 11952963
+2171 142266931
+2172 63800323
+2173 140744598
+2174 89649673
+2175 152470976
+2176 43383
+2177 39263661
+2178 411002
+2179 84492190
+2180 24470845
+2181 102456000
+2182 47457739
+2183 126078295
+2184 1013394
+2185 77029197
+2186 4471597
+2187 118855872
+2188 45135873
+2189 125474980
+2190 69892571
+2191 143710048
+2192 115293
+2193 59576930
+2194 995537
+2195 103955764
+2196 34743010
+2197 116341158
+2198 58239452
+2199 136542962
+2200 2302542
+2201 98978702
+2202 8995018
+2203 134284290
+2204 56939156
+2205 134830244
+2206 82705434
+2207 148793193
+2208 150765
+2209 48730052
+2210 1230270
+2211 90512132
+2212 29259164
+2213 107583767
+2214 51446486
+2215 128433785
+2216 1150230
+2217 86816160
+2218 5371747
+2219 123057095
+2220 49396563
+2221 129127656
+2222 74946901
+2223 145933533
+2224 376709
+2225 67879848
+2226 2621339
+2227 106379416
+2228 38951539
+2229 119842421
+2230 62513965
+2231 135958284
+2232 2553200
+2233 106883629
+2234 10619260
+2235 140153140
+2236 61563673
+2237 137448424
+2238 87265684
+2239 149279657
+2240 239491
+2241 57218391
+2242 1766295
+2243 100559934
+2244 34305465
+2245 113606780
+2246 56959277
+2247 132035577
+2248 1261494
+2249 94129845
+2250 5967315
+2251 128412455
+2252 53604191
+2253 129929026
+2254 77263888
+2255 147354036
+2256 583224
+2257 76647238
+2258 3802573
+2259 115801104
+2260 45163854
+2261 123675005
+2262 68379476
+2263 138755787
+2264 2899890
+2265 112476388
+2266 12054541
+2267 141710858
+2268 65930021
+2269 141958107
+2270 91062705
+2271 154106165
+2272 653772
+2273 65588601
+2274 4109532
+2275 104064264
+2276 39061480
+2277 117016055
+2278 63426813
+2279 135467580
+2280 1676456
+2281 99519583
+2282 8403088
+2283 133865809
+2284 57083022
+2285 132631333
+2286 82256852
+2287 149832018
+2288 1424066
+2289 83066825
+2290 7765505
+2291 120475696
+2292 49679924
+2293 126938414
+2294 74105398
+2295 140801362
+2296 3718454
+2297 119135433
+2298 15946010
+2299 146224736
+2300 70148221
+2301 142104933
+2302 96908839
+2303 154958056
+2304 14668
+2305 17710389
+2306 152339
+2307 61146735
+2308 13732533
+2309 90835174
+2310 35444430
+2311 116466781
+2312 1260616
+2313 64410707
+2314 4937216
+2315 108449199
+2316 38277941
+2317 117443097
+2318 64843607
+2319 138452662
+2320 52176
+2321 34204242
+2322 436722
+2323 82606724
+2324 22573114
+2325 103729882
+2326 47080639
+2327 126436720
+2328 2682015
+2329 85118611
+2330 9435654
+2331 125944194
+2332 50647240
+2333 129778613
+2334 76996849
+2335 146263652
+2336 69377
+2337 37104209
+2338 588663
+2339 83520402
+2340 24475142
+2341 103244272
+2342 47687296
+2343 127559194
+2344 1331598
+2345 75930140
+2346 5415786
+2347 117728924
+2348 45678017
+2349 124174235
+2350 71183471
+2351 143058177
+2352 186071
+2353 56437806
+2354 1367660
+2355 103700761
+2356 34386372
+2357 117161023
+2358 59736175
+2359 133747983
+2360 2858417
+2361 97791974
+2362 10391163
+2363 131898874
+2364 58101536
+2365 132432762
+2366 82522443
+2367 151687672
+2368 104359
+2369 41456194
+2370 790346
+2371 88686821
+2372 26691937
+2373 107021017
+2374 50531362
+2375 127309226
+2376 1461417
+2377 82359184
+2378 6191439
+2379 120544277
+2380 48268200
+2381 128982575
+2382 74479582
+2383 143907328
+2384 278912
+2385 62650843
+2386 1788613
+2387 109484800
+2388 37529899
+2389 117701107
+2390 61337841
+2391 136051612
+2392 3231099
+2393 103832323
+2394 11985510
+2395 136039587
+2396 62632023
+2397 135123916
+2398 86208609
+2399 148586372
+2400 376683
+2401 55664253
+2402 2317046
+2403 99899322
+2404 34438393
+2405 112967205
+2406 57994305
+2407 131627917
+2408 1676250
+2409 90463878
+2410 7227855
+2411 127804117
+2412 53805431
+2413 132103563
+2414 78104919
+2415 143270132
+2416 850466
+2417 76920281
+2418 4586905
+2419 116840045
+2420 45485855
+2421 125698265
+2422 69907656
+2423 139989131
+2424 3591859
+2425 112044567
+2426 13888317
+2427 142815326
+2428 66150096
+2429 139567154
+2430 90548743
+2431 150276135
+2432 81959
+2433 40935625
+2434 685066
+2435 85795329
+2436 26541294
+2437 105937421
+2438 49369198
+2439 125730879
+2440 1355339
+2441 80048870
+2442 5384137
+2443 121983603
+2444 47467062
+2445 125698043
+2446 71882909
+2447 140770178
+2448 203883
+2449 61333524
+2450 1543202
+2451 106667809
+2452 36254418
+2453 117017362
+2454 60228564
+2455 136240589
+2456 2854305
+2457 102273023
+2458 10466497
+2459 134877724
+2460 59565228
+2461 137242153
+2462 84018620
+2463 150521373
+2464 256442
+2465 50477471
+2466 1753857
+2467 91888278
+2468 30925017
+2469 108011562
+2470 54282398
+2471 130166879
+2472 1508218
+2473 89534853
+2474 6544535
+2475 126685269
+2476 51444446
+2477 130666832
+2478 75895272
+2479 141711293
+2480 592086
+2481 68911784
+2482 3429824
+2483 109141967
+2484 40924396
+2485 120240847
+2486 65368053
+2487 135068011
+2488 3247228
+2489 108343400
+2490 12296645
+2491 139200103
+2492 64253946
+2493 139464434
+2494 90144018
+2495 150933892
+2496 387203
+2497 60112680
+2498 2471124
+2499 101364240
+2500 36060090
+2501 113115258
+2502 59092846
+2503 133660255
+2504 1688067
+2505 95166867
+2506 7464938
+2507 130201426
+2508 55435813
+2509 131378360
+2510 80339270
+2511 144343901
+2512 891695
+2513 79403496
+2514 4846349
+2515 117591013
+2516 46701564
+2517 127372321
+2518 71594029
+2519 139752761
+2520 3585107
+2521 114633188
+2522 14060404
+2523 142634270
+2524 68291733
+2525 141838947
+2526 92319322
+2527 154201069
+2528 930345
+2529 66936301
+2530 5057724
+2531 108112438
+2532 41230837
+2533 117236757
+2534 63114220
+2535 134889085
+2536 2264842
+2537 102766997
+2538 10189124
+2539 134349467
+2540 59607139
+2541 136132248
+2542 84365276
+2543 145396604
+2544 1934988
+2545 87496829
+2546 8963586
+2547 123821148
+2548 51594867
+2549 126583574
+2550 75453026
+2551 143429208
+2552 4688291
+2553 121133074
+2554 18369851
+2555 146651284
+2556 71294421
+2557 144513528
+2558 98980071
+2559 156474703
+2560 574681
+2561 72013501
+2562 4170731
+2563 108740256
+2564 42160291
+2565 118802622
+2566 65199657
+2567 134318813
+2568 1711436
+2569 106364466
+2570 9377548
+2571 136089932
+2572 61781874
+2573 136063027
+2574 85955728
+2575 149722927
+2576 1460336
+2577 90368418
+2578 8479930
+2579 123461144
+2580 52938120
+2581 127696495
+2582 77264195
+2583 140512222
+2584 4006765
+2585 123742005
+2586 17570016
+2587 149293906
+2588 74254766
+2589 142980282
+2590 99639510
+2591 155578996
+2592 1511044
+2593 78158991
+2594 9250811
+2595 113168463
+2596 47728382
+2597 121344153
+2598 72033721
+2599 140215256
+2600 2570305
+2601 111183745
+2602 13333752
+2603 139295086
+2604 65791739
+2605 139747945
+2606 91745662
+2607 150998682
+2608 3444869
+2609 95018873
+2610 16736273
+2611 129851414
+2612 59218692
+2613 131000976
+2614 84000432
+2615 145265002
+2616 5672362
+2617 129042313
+2618 24831279
+2619 154064778
+2620 79250112
+2621 148089275
+2622 105806379
+2623 159746592
+2624 2109900
+2625 81321389
+2626 11843659
+2627 115581441
+2628 51249627
+2629 121796815
+2630 74697826
+2631 140540844
+2632 3240333
+2633 114853705
+2634 16467288
+2635 141781340
+2636 68579437
+2637 140809504
+2638 94904134
+2639 155028084
+2640 4586544
+2641 99753752
+2642 20819775
+2643 130425050
+2644 63177958
+2645 133283733
+2646 89252057
+2647 148817311
+2648 7053685
+2649 132802587
+2650 29207405
+2651 157030973
+2652 82807923
+2653 147127777
+2654 108975046
+2655 159194852
+2656 4445525
+2657 87538245
+2658 20784213
+2659 118896880
+2660 57340871
+2661 126578306
+2662 82097782
+2663 143862076
+2664 5584146
+2665 118805371
+2666 25221541
+2667 147671549
+2668 74271136
+2669 143730494
+2670 101333431
+2671 159741857
+2672 8851766
+2673 106132315
+2674 33077565
+2675 135998977
+2676 69673935
+2677 138169687
+2678 97158858
+2679 152019357
+2680 11056251
+2681 137568589
+2682 41686204
+2683 158704164
+2684 88686482
+2685 153232585
+2686 119340173
+2687 166854230
+2688 1636939
+2689 79441694
+2690 10758207
+2691 114454589
+2692 48997170
+2693 121025473
+2694 73788585
+2695 140017108
+2696 2803974
+2697 113091943
+2698 15035959
+2699 143213303
+2700 67542300
+2701 139865148
+2702 94357868
+2703 151251593
+2704 3949896
+2705 96756463
+2706 19757814
+2707 127100039
+2708 61307563
+2709 133125189
+2710 86228962
+2711 148212827
+2712 6212546
+2713 131017681
+2714 27930582
+2715 151134460
+2716 82373955
+2717 146627678
+2718 108157839
+2719 161615558
+2720 4214631
+2721 85948560
+2722 20705208
+2723 119264661
+2724 55552169
+2725 124259355
+2726 81871855
+2727 142802420
+2728 5277704
+2729 118871743
+2730 25065926
+2731 145881513
+2732 73629648
+2733 144891803
+2734 101051234
+2735 156434054
+2736 8345645
+2737 103642148
+2738 33131740
+2739 135995086
+2740 69281917
+2741 139691917
+2742 95699348
+2743 155369212
+2744 10648845
+2745 137233318
+2746 41054643
+2747 161076108
+2748 89238030
+2749 153490810
+2750 116768211
+2751 165921864
+2752 4930729
+2753 89048703
+2754 23605140
+2755 120634134
+2756 58062708
+2757 128107941
+2758 83936018
+2759 144465779
+2760 6018705
+2761 121923012
+2762 27889312
+2763 146495022
+2764 77138497
+2765 144984094
+2766 104193266
+2767 157293335
+2768 10010204
+2769 104420755
+2770 37849428
+2771 138752421
+2772 72704805
+2773 139749859
+2774 100078206
+2775 154595744
+2776 12221109
+2777 139139719
+2778 45853920
+2779 162191363
+2780 92493968
+2781 155114895
+2782 120804768
+2783 166649637
+2784 10508975
+2785 95266740
+2786 37787523
+2787 126247609
+2788 67366579
+2789 131265716
+2790 94198384
+2791 149777177
+2792 11574993
+2793 127074826
+2794 42907803
+2795 154953176
+2796 85512056
+2797 149108134
+2798 112059991
+2799 163299205
+2800 18591730
+2801 113565050
+2802 54596835
+2803 142248590
+2804 82610665
+2805 147555374
+2806 109980262
+2807 161349248
+2808 20935997
+2809 146140513
+2810 62193161
+2811 167341203
+2812 100511596
+2813 161199461
+2814 132696320
+2815 173113372
+2816 932098
+2817 73705793
+2818 5502116
+2819 109501785
+2820 43830184
+2821 119884101
+2822 67377838
+2823 135458192
+2824 2433685
+2825 108299931
+2826 11187411
+2827 138677489
+2828 63177309
+2829 135039974
+2830 88575169
+2831 148462632
+2832 2099186
+2833 91644863
+2834 10402545
+2835 123438153
+2836 55789068
+2837 128257948
+2838 79511331
+2839 143224007
+2840 5180676
+2841 126794604
+2842 20185085
+2843 149019808
+2844 75889862
+2845 147076860
+2846 100530181
+2847 156047939
+2848 2189646
+2849 80418880
+2850 10974090
+2851 114024031
+2852 49385284
+2853 120984582
+2854 73806199
+2855 137143764
+2856 3487905
+2857 113058554
+2858 15835036
+2859 142433293
+2860 67052820
+2861 139186514
+2862 93483191
+2863 152104699
+2864 4460768
+2865 97221285
+2866 18890399
+2867 129429882
+2868 60588477
+2869 131574867
+2870 85670376
+2871 147281730
+2872 7220349
+2873 131534709
+2874 27590271
+2875 154379106
+2876 82133206
+2877 146742055
+2878 107964112
+2879 160865797
+2880 2918655
+2881 83271649
+2882 13792791
+2883 116183089
+2884 52800211
+2885 124907760
+2886 77753669
+2887 141152439
+2888 4324336
+2889 116738096
+2890 18630462
+2891 144629016
+2892 70294035
+2893 139799710
+2894 97193310
+2895 152565537
+2896 5872337
+2897 100103262
+2898 23077597
+2899 131420534
+2900 63762812
+2901 134538502
+2902 89113924
+2903 149626971
+2904 8898348
+2905 133046197
+2906 31919716
+2907 155554649
+2908 84411661
+2909 150932008
+2910 110315168
+2911 160806197
+2912 5586151
+2913 88394234
+2914 22724402
+2915 120775622
+2916 58559218
+2917 125781349
+2918 83317030
+2919 144643371
+2920 6824151
+2921 119523033
+2922 27518088
+2923 147717490
+2924 74961984
+2925 143160327
+2926 104434713
+2927 154217660
+2928 10316210
+2929 106559793
+2930 34746065
+2931 137142638
+2932 72468402
+2933 138578200
+2934 97381392
+2935 152493106
+2936 13102378
+2937 138696090
+2938 44108163
+2939 158160891
+2940 91667921
+2941 155595728
+2942 119431285
+2943 169130090
+2944 2459776
+2945 82018742
+2946 12619666
+2947 114673305
+2948 51264060
+2949 121450095
+2950 75216592
+2951 140219369
+2952 3832509
+2953 114922638
+2954 17685242
+2955 142659030
+2956 69266767
+2957 139779462
+2958 94511031
+2959 153169181
+2960 5075213
+2961 98753536
+2962 21902697
+2963 130770037
+2964 62594333
+2965 133307792
+2966 88148296
+2967 149733699
+2968 7890118
+2969 133358775
+2970 30457713
+2971 154517030
+2972 82694085
+2973 149085761
+2974 110386899
+2975 162252240
+2976 5259600
+2977 87806906
+2978 22507445
+2979 119466498
+2980 57691083
+2981 127707815
+2982 82756572
+2983 144327024
+2984 6629992
+2985 120842226
+2986 27524276
+2987 149509918
+2988 75671990
+2989 143701181
+2990 101564735
+2991 158538548
+2992 9864960
+2993 105532019
+2994 35020301
+2995 134985234
+2996 71764499
+2997 139833958
+2998 98352927
+2999 155659614
+3001 139895979
+3002 44146910
+3003 160456800
+3004 91124233
+3005 154702227
+3006 118797971
+3007 165294928
+3008 6120589
+3009 89990599
+3010 26066681
+3011 121371393
+3012 59895248
+3013 128435485
+3014 85609465
+3015 145632698
+3016 7508838
+3017 124043698
+3018 30719698
+3019 148239576
+3020 78652923
+3021 145656165
+3022 105904822
+3023 158329638
+3024 11695663
+3025 108843684
+3026 40286674
+3027 139249337
+3028 74942420
+3029 140565207
+3030 102277201
+3031 155999482
+3032 14623104
+3033 141133804
+3034 48692746
+3035 163882184
+3036 95294425
+3037 156150093
+3038 121729991
+3039 167797321
+3040 12291581
+3041 97604203
+3042 40110999
+3043 127700624
+3044 68699258
+3045 134376636
+3046 94907005
+3047 150694435
+3048 13187179
+3049 129803067
+3050 44804540
+3051 156295692
+3052 86093792
+3053 150835451
+3054 114573263
+3055 162880486
+3056 20622931
+3057 113878196
+3058 56577971
+3059 145053366
+3060 84172186
+3061 145584375
+3062 110457617
+3063 161543161
+3064 23289574
+3065 148412254
+3066 63970562
+3067 169267050
+3068 103908901
+3069 161167731
+3070 131329206
+3071 175230378
+3072 201255
+3073 59731483
+3074 1689145
+3075 101838541
+3076 35442161
+3077 115374765
+3078 59385205
+3079 132757681
+3080 1249145
+3081 93840481
+3082 6199928
+3083 131414132
+3084 53647983
+3085 131373452
+3086 78174796
+3087 148119850
+3088 542664
+3089 79621263
+3090 3876917
+3091 119105159
+3092 46586724
+3093 123445850
+3094 69698852
+3095 138814459
+3096 2850905
+3097 114957027
+3098 12531555
+3099 144048258
+3100 66681331
+3101 139011208
+3102 91800004
+3103 154175942
+3104 501443
+3105 69259471
+3106 3722407
+3107 106654913
+3108 40997621
+3109 118200849
+3110 64188238
+3111 137675699
+3112 1486697
+3113 102261156
+3114 7674235
+3115 134328787
+3116 58868707
+3117 134984447
+3118 83082004
+3119 146802999
+3120 1265809
+3121 88129789
+3122 7583345
+3123 122704229
+3124 51159928
+3125 126733648
+3126 75364346
+3127 142255862
+3128 3467852
+3129 121589303
+3130 15482688
+3131 147039130
+3132 71066829
+3133 143951449
+3134 96175217
+3135 153034960
+3136 861281
+3137 72392234
+3138 5834356
+3139 108865563
+3140 43268392
+3141 118524644
+3142 66502263
+3143 137807945
+3144 2045323
+3145 104683309
+3146 10566776
+3147 137770224
+3148 60762343
+3149 136609754
+3150 85098922
+3151 150234894
+3152 2140794
+3153 91492088
+3154 11452116
+3155 124736891
+3156 54407860
+3157 129707429
+3158 79445660
+3159 143893689
+3160 4684592
+3161 123585345
+3162 20155767
+3163 147059328
+3164 74657088
+3165 145920539
+3166 99082967
+3167 157018711
+3168 1874244
+3169 79482608
+3170 10525107
+3171 114509845
+3172 47568341
+3173 121695906
+3174 73006780
+3175 139854248
+3176 2865098
+3177 112490058
+3178 14492926
+3179 138201457
+3180 65175254
+3181 139228334
+3182 89368342
+3183 151098033
+3184 4027295
+3185 96408561
+3186 18317287
+3187 128917773
+3188 60302029
+3189 132214740
+3190 85437184
+3191 147411661
+3192 6261812
+3193 129089675
+3194 26541042
+3195 153457383
+3196 79407235
+3197 147375924
+3198 105540174
+3199 158537902
+3200 254560
+3201 61140942
+3202 2079586
+3203 102273493
+3204 36013453
+3205 114946694
+3206 59477294
+3207 132402234
+3208 1267132
+3209 94864402
+3210 6231016
+3211 128860857
+3212 55510689
+3213 134523471
+3214 79469469
+3215 144267066
+3216 689334
+3217 80999563
+3218 4741176
+3219 120298631
+3220 46729942
+3221 124911553
+3222 71283833
+3223 142858323
+3224 2917360
+3225 113050357
+3226 12637530
+3227 143188891
+3228 65947496
+3229 141674174
+3230 91921576
+3231 149943922
+3232 510696
+3233 69603870
+3234 3712297
+3235 107292850
+3236 40932153
+3237 119204847
+3238 63187269
+3239 134164175
+3240 1487970
+3241 102514083
+3242 7748745
+3243 133204930
+3244 58322635
+3245 135192943
+3246 83430319
+3247 146396759
+3248 1232255
+3249 87574417
+3250 7651678
+3251 122299134
+3252 51608729
+3253 128652865
+3254 75676785
+3255 142815163
+3256 3468854
+3257 117844144
+3258 15560264
+3259 147203671
+3260 71250379
+3261 143672438
+3262 96002950
+3263 155727097
+3264 1048655
+3265 74155079
+3266 6499011
+3267 110034194
+3268 43914601
+3269 118867445
+3270 67227351
+3271 135365814
+3272 2023563
+3273 105774570
+3274 10504641
+3275 135348409
+3276 60849890
+3277 135709282
+3278 86824366
+3279 150693829
+3280 2469269
+3281 90725508
+3282 12271089
+3283 124976308
+3284 55447798
+3285 129375506
+3286 79498401
+3287 144975788
+3288 4690572
+3289 123486373
+3290 19895041
+3291 149404708
+3292 75273636
+3293 147424425
+3294 101025328
+3295 156058249
+3296 1821572
+3297 79106703
+3298 10467740
+3299 112424288
+3300 48199031
+3301 120802354
+3302 71647258
+3303 138326952
+3304 2832315
+3305 111856440
+3306 14662727
+3307 141361472
+3308 64370479
+3309 139523791
+3310 90738322
+3311 150851981
+3312 4065794
+3313 98065847
+3314 18544144
+3315 129932639
+3316 60550138
+3317 132351196
+3318 85055106
+3319 147011736
+3320 6176123
+3321 127322362
+3322 26055854
+3323 151150982
+3324 78358984
+3325 149599777
+3326 106664741
+3327 161191718
+3328 350593
+3329 62615146
+3330 2475087
+3331 105515755
+3332 37175376
+3333 115242950
+3334 60368793
+3335 132396982
+3336 1699513
+3337 95617775
+3338 7787453
+3339 133223916
+3340 56259828
+3341 133484737
+3342 81377286
+3343 149232501
+3344 858869
+3345 82127696
+3346 5179638
+3347 120611324
+3348 47840005
+3349 125486557
+3350 71196706
+3351 139346777
+3352 3698770
+3353 117057797
+3354 15010389
+3355 145651686
+3356 68464703
+3357 142337481
+3358 94123522
+3359 152664625
+3360 810868
+3361 71435120
+3362 4884725
+3363 108137227
+3364 42669279
+3365 118935892
+3366 65940242
+3367 133387634
+3368 2087068
+3369 103438507
+3370 9642219
+3371 134688226
+3372 59815905
+3373 136729236
+3374 84315427
+3375 148160277
+3376 1834089
+3377 90884068
+3378 9333392
+3379 124007072
+3380 53305703
+3381 128876427
+3382 77547121
+3383 143821763
+3384 4505074
+3385 123088299
+3386 17648934
+3387 146362250
+3388 73780102
+3389 143504508
+3390 98199359
+3391 157873555
+3392 1337048
+3393 74881950
+3394 7264025
+3395 109793559
+3396 45830866
+3397 120901486
+3398 67681752
+3399 138388219
+3400 2816706
+3401 107402506
+3402 12765570
+3403 137283378
+3404 62752866
+3405 138156137
+3406 88580702
+3407 148944346
+3408 2987142
+3409 92819040
+3410 13096788
+3411 127400556
+3412 56565901
+3413 130756832
+3414 79908377
+3415 143567227
+3416 6082836
+3417 124942291
+3418 22532602
+3419 151346490
+3420 76952048
+3421 144749367
+3422 102343617
+3423 160225164
+3424 2627108
+3425 79629865
+3426 12164042
+3427 115416910
+3428 49005476
+3429 120791323
+3430 74641972
+3431 141840413
+3432 3821445
+3433 112556854
+3434 16913159
+3435 142643053
+3436 67970843
+3437 139957281
+3438 92610573
+3439 152762121
+3440 5183868
+3441 99146577
+3442 20499242
+3443 130410802
+3444 62158814
+3445 131570277
+3446 87977786
+3447 146079192
+3448 7860273
+3449 131888627
+3450 29380094
+3451 153231401
+3452 82532063
+3453 149181432
+3454 108077797
+3455 160061719
+3456 448558
+3457 63997628
+3458 3051379
+3459 106948567
+3460 38635225
+3461 116580574
+3462 62407879
+3463 133515235
+3464 1708070
+3465 95151317
+3466 7819865
+3467 131643934
+3468 55533763
+3469 131034853
+3470 81019985
+3471 149347855
+3472 1089455
+3473 84404068
+3474 6209900
+3475 120399838
+3476 48719388
+3477 128661147
+3478 73019135
+3479 141018145
+3480 3707767
+3481 116847386
+3482 14903533
+3483 143973354
+3484 69185726
+3485 142333567
+3486 94771167
+3487 150723344
+3488 817452
+3489 71396363
+3490 4931212
+3491 110286309
+3492 42883103
+3493 119084740
+3494 64387696
+3495 136636787
+3496 2055186
+3497 103627193
+3498 9500925
+3499 135086433
+3500 60041609
+3501 134632091
+3502 84624888
+3503 148490588
+3504 1858365
+3505 89686444
+3506 9305349
+3507 124344509
+3508 53413734
+3509 128872221
+3510 78558083
+3511 142207216
+3512 4491162
+3513 121922352
+3514 17784671
+3515 148211704
+3516 72578771
+3517 146768726
+3518 97314728
+3519 157049318
+3520 1580102
+3521 75159932
+3522 7962971
+3523 111114388
+3524 45461001
+3525 120296287
+3526 68775879
+3527 137697161
+3528 2869451
+3529 107741463
+3530 12607076
+3531 137331688
+3532 62805277
+3533 138370160
+3534 88923323
+3535 150257590
+3536 3335540
+3537 93223854
+3538 14000270
+3539 127710180
+3540 56658490
+3541 129786341
+3542 81952551
+3543 143096923
+3544 6000400
+3545 125142371
+3546 22581786
+3547 149624640
+3548 76611832
+3549 145766276
+3550 102217662
+3551 157994443
+3552 2575183
+3553 80831058
+3554 12200074
+3555 114103951
+3556 49804376
+3557 124017676
+3558 73639089
+3559 140005200
+3560 3884995
+3561 114574972
+3562 16723532
+3563 141109021
+3564 66556403
+3565 140322656
+3566 93046065
+3567 152324952
+3568 5168347
+3569 97021723
+3570 20559333
+3571 131608906
+3572 61290469
+3573 133579343
+3574 86967466
+3575 148169247
+3576 7875825
+3577 130424793
+3578 29377930
+3579 152507544
+3580 81322755
+3581 149718803
+3582 109354073
+3583 160127036
+3584 3726135
+3585 91888578
+3586 22805448
+3587 122842903
+3588 59706806
+3589 130232409
+3590 87703611
+3591 142673584
+3592 4872154
+3593 122704994
+3594 26785682
+3595 149257786
+3596 78509420
+3597 145061395
+3598 103470785
+3599 158630687
+3600 8794472
+3601 111443442
+3602 38080921
+3603 139507784
+3604 74551970
+3605 141079438
+3606 101812836
+3607 156351010
+3608 11476360
+3609 142926106
+3610 46322583
+3611 161946524
+3612 94719905
+3613 157007824
+3614 121211705
+3615 165873804
+3616 8188726
+3617 99290499
+3618 36625361
+3619 127939220
+3620 68270127
+3621 133202894
+3622 93523749
+3623 150951776
+3624 9182722
+3625 128432957
+3626 40984717
+3627 153625950
+3628 84549231
+3629 150253640
+3630 111900940
+3631 161050892
+3632 16209191
+3633 114985361
+3634 54419135
+3635 147726743
+3636 83483363
+3637 146137758
+3638 111762010
+3639 161710373
+3640 18336638
+3641 148728085
+3642 63205736
+3643 168326815
+3644 102105951
+3645 160896705
+3646 132399544
+3647 174609400
+3648 10714844
+3649 101869528
+3650 42693587
+3651 130497073
+3652 72196740
+3653 132499698
+3654 98929648
+3655 153139228
+3656 11926795
+3657 131041879
+3658 47612221
+3659 156795719
+3660 88459247
+3661 150765295
+3662 117091664
+3663 165541260
+3664 21111486
+3665 119884532
+3666 61555467
+3667 147577354
+3668 89493423
+3669 149441682
+3670 116510728
+3671 164060933
+3672 23041575
+3673 152359899
+3674 71426527
+3675 169450215
+3676 107145617
+3677 163121178
+3678 136452524
+3679 177584037
+3680 19328205
+3681 107805060
+3682 59719884
+3683 136212434
+3684 80650901
+3685 138835798
+3686 108352478
+3687 157489665
+3688 20802749
+3689 138228385
+3690 63967180
+3691 162472890
+3692 98479953
+3693 155841859
+3694 127950472
+3695 172055482
+3696 33743981
+3697 127576808
+3698 78051912
+3699 154280783
+3700 99061038
+3701 155764227
+3702 125692192
+3703 169270871
+3704 35480592
+3705 157438506
+3706 86461028
+3707 178810281
+3708 118409088
+3709 170976241
+3710 145852243
+3711 183822189
+3712 3735859
+3713 91686737
+3714 22895422
+3715 123688127
+3716 59775111
+3717 127487333
+3718 86088421
+3719 145517426
+3720 4898731
+3721 123414169
+3722 26582530
+3723 150339150
+3724 75899398
+3725 146300399
+3726 104432363
+3727 157670963
+3728 8855984
+3729 110673280
+3730 39149334
+3731 139460356
+3732 74793328
+3733 138165158
+3734 101512017
+3735 158630201
+3736 11141718
+3737 142118103
+3738 47434191
+3739 162686098
+3740 93208144
+3741 155014052
+3742 124793588
+3743 169404304
+3744 8186836
+3745 97763296
+3746 36170716
+3747 128470561
+3748 67106265
+3749 132354207
+3750 95264332
+3751 150705101
+3752 9231275
+3753 129105020
+3754 40848945
+3755 153322838
+3756 85578785
+3757 148962913
+3758 111322308
+3759 159842536
+3760 16372892
+3761 116748506
+3762 54446528
+3763 143690555
+3764 83618046
+3765 146427155
+3766 113319840
+3767 161963596
+3768 18477817
+3769 147449384
+3770 62683945
+3771 167252968
+3772 102779223
+3773 161514717
+3774 132380572
+3775 172394265
+3776 10956812
+3777 101102074
+3778 43002053
+3779 130205756
+3780 71299226
+3781 133920023
+3782 99038459
+3783 151777016
+3784 12088141
+3785 132592265
+3786 47608058
+3787 158079280
+3788 88290584
+3789 151867427
+3790 116615227
+3791 162490543
+3792 20850605
+3793 118248660
+3794 62870898
+3795 149571216
+3796 89090723
+3797 148092040
+3798 114054838
+3799 163913391
+3800 23445531
+3801 149794142
+3802 70882460
+3803 171875921
+3804 107900818
+3805 163115546
+3806 135721673
+3807 180067125
+3808 19699492
+3809 108070773
+3810 59910687
+3811 135935221
+3812 80568288
+3813 141276298
+3814 108819382
+3815 157845656
+3816 20514116
+3817 137746409
+3818 63866545
+3819 161378691
+3820 97766135
+3821 157558868
+3822 127330397
+3823 171548999
+3824 33555620
+3825 124700332
+3826 77630536
+3827 155131475
+3828 98890344
+3829 155391062
+3830 126564270
+3831 170709292
+3832 35510149
+3833 159262817
+3834 86808777
+3835 179591788
+3836 119007141
+3837 171607763
+3838 145185651
+3839 179584537
+3840 5248834
+3841 94381395
+3842 25142684
+3843 125029977
+3844 62374088
+3845 128849043
+3846 87871804
+3847 144188233
+3848 6665412
+3849 127671610
+3850 29784285
+3851 149529628
+3852 79920105
+3853 144288880
+3854 106776349
+3855 157779438
+3856 10815298
+3857 113220237
+3858 40445543
+3859 140762877
+3860 75773653
+3861 140632143
+3862 103623603
+3863 157234398
+3864 13664172
+3865 141496529
+3866 49047139
+3867 164327348
+3868 96423815
+3869 156918208
+3870 123983494
+3871 169639736
+3872 9955276
+3873 98929189
+3874 38782591
+3875 129402611
+3876 69690028
+3877 132179393
+3878 96671076
+3879 149545854
+3880 11299540
+3881 131048249
+3882 43683026
+3883 153593878
+3884 86980793
+3885 150579225
+3886 113721479
+3887 163433378
+3888 18288118
+3889 117626580
+3890 57268114
+3891 147686268
+3892 85663912
+3893 149237379
+3894 114987586
+3895 161312153
+3896 21048857
+3897 148158173
+3898 65424235
+3899 169280130
+3900 103771906
+3901 161537030
+3902 134288168
+3903 173792979
+3904 12960511
+3905 102519133
+3906 45369427
+3907 130691609
+3908 74068241
+3909 134680537
+3910 100765318
+3911 154124084
+3912 14381256
+3913 132503803
+3914 50468497
+3915 155422899
+3916 89987601
+3917 153357094
+3918 119133919
+3919 164333818
+3920 23082249
+3921 119365524
+3922 64705274
+3923 151757145
+3924 90824050
+3925 152245676
+3926 115777440
+3927 163515343
+3928 26156712
+3929 150905780
+3930 73560531
+3931 172536320
+3932 108057388
+3933 165768765
+3934 141263878
+3935 176260371
+3936 21942088
+3937 107555201
+3938 62268749
+3939 138199607
+3940 82465961
+3941 140193308
+3942 110310594
+3943 157889133
+3944 22933325
+3945 142090051
+3946 66132058
+3947 161192310
+3948 99645335
+3949 157021730
+3950 128658914
+3951 171411723
+3952 35270945
+3953 127187831
+3954 80057312
+3955 156845398
+3956 101331709
+3957 156490842
+3958 128083673
+3959 170342057
+3960 38086484
+3961 160501835
+3962 89844274
+3963 177897904
+3964 120122911
+3965 170382486
+3966 148410215
+3967 184966002
+3968 5237981
+3969 93411871
+3970 25319643
+3971 123864599
+3972 62432373
+3973 126750787
+3974 87295995
+3975 143712293
+3976 6594523
+3977 123879692
+3978 29920319
+3979 149917197
+3980 78206479
+3981 147166497
+3982 106047644
+3983 157156928
+3984 10859551
+3985 111691920
+3986 40631403
+3987 140513806
+3988 76634426
+3989 139063133
+3990 103771092
+3991 153582149
+3992 13641885
+3993 143790093
+3994 49695303
+3995 163514194
+3996 96529521
+3997 155556407
+3998 123318955
+3999 169289615
+4000 10112943
+4001 98439518
+4002 38567208
+4003 129333738
+4004 69035501
+4005 133651770
+4006 96969489
+4007 150635675
+4008 11249013
+4009 131233435
+4010 43363397
+4011 156228080
+4012 86040917
+4013 151548517
+4014 114912316
+4015 160874557
+4016 18240181
+4017 118555607
+4018 56463158
+4019 147103894
+4020 85801277
+4021 146905477
+4022 114321362
+4023 162251141
+4024 21145968
+4025 146676116
+4026 65586396
+4027 169390668
+4028 102946385
+4029 160439373
+4030 132905650
+4031 173373924
+4032 12954347
+4033 102563760
+4034 45569611
+4035 131327590
+4036 72940525
+4037 134777311
+4038 101461656
+4039 152372568
+4040 14264745
+4041 132941325
+4042 50226554
+4043 156912202
+4044 90498278
+4045 153052604
+4046 118498746
+4047 166768968
+4048 23284733
+4049 120538340
+4050 64549010
+4051 150848894
+4052 90680852
+4053 147981786
+4054 117662854
+4055 164602162
+4056 26085488
+4057 152423406
+4058 73920499
+4059 170538392
+4060 109528224
+4061 167335477
+4062 137072713
+4063 175327679
+4064 21847541
+4065 108696160
+4066 61227549
+4067 136930815
+4068 83359820
+4069 141490316
+4070 110275654
+4071 158333630
+4072 23189254
+4073 139009910
+4074 64714534
+4075 164912506
+4076 101215697
+4077 157685236
+4078 128683417
+4079 171643294
+4080 35611916
+4081 129566457
+4082 79616243
+4083 153844045
+4084 101873461
+4085 157591516
+4086 127786211
+4087 172202256
+4088 38748208
+4089 159566906
+4090 87797284
+4091 176448362
+4092 120944029
+4093 171350081
+4094 146605383
+4095 183301627


[04/22] kylin git commit: APACHE-KYLIN-2729: add basic support classes for cube planner algorithms

Posted by nj...@apache.org.
APACHE-KYLIN-2729: add basic support classes for cube planner algorithms


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03e45864
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03e45864
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03e45864

Branch: refs/heads/yaho-cube-planner
Commit: 03e458648cb3800b6ecfb507f4cf3c1b8e547280
Parents: b6ec7e9
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 14:57:29 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:38 2017 +0800

----------------------------------------------------------------------
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  |    3 +-
 .../algorithm/AbstractRecommendAlgorithm.java   |   82 +
 .../cube/cuboid/algorithm/BPUSCalculator.java   |  165 +
 .../cube/cuboid/algorithm/BenefitPolicy.java    |   41 +
 .../cuboid/algorithm/CuboidBenefitModel.java    |  210 +
 .../algorithm/CuboidRecommendAlgorithm.java     |   49 +
 .../cube/cuboid/algorithm/CuboidStats.java      |  272 ++
 .../cube/cuboid/algorithm/CuboidStatsUtil.java  |  166 +
 .../cube/cuboid/algorithm/PBPUSCalculator.java  |   53 +
 .../cube/cuboid/algorithm/SPBPUSCalculator.java |   41 +
 .../cuboid/algorithm/AlgorithmTestBase.java     |  310 ++
 .../cuboid/algorithm/CuboidStatsUtilTest.java   |  161 +
 core-cube/src/test/resources/statistics.txt     | 4092 ++++++++++++++++++
 13 files changed, 5643 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
index 414fc4a..0cf770d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -88,8 +88,7 @@ public class TreeCuboidScheduler extends CuboidScheduler {
             return createFromCuboids(allCuboidIds, Cuboid.cuboidSelectComparator);
         }
 
-        @VisibleForTesting
-        static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+        public static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
             // sort the cuboid ids in descending order, so that don't need to adjust
             // the cuboid tree when adding cuboid id to the tree.
             Collections.sort(allCuboidIds, new Comparator<Long>() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
new file mode 100755
index 0000000..b35c738
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractRecommendAlgorithm implements CuboidRecommendAlgorithm {
+    private static final Logger logger = LoggerFactory.getLogger(AbstractRecommendAlgorithm.class);
+
+    private CuboidStats cuboidStats;
+    private BenefitPolicy benefitPolicy;
+
+    private AtomicBoolean cancelRequested = new AtomicBoolean(false);
+    private AtomicBoolean canceled = new AtomicBoolean(false);
+
+    private long timeoutMillis;
+
+    public AbstractRecommendAlgorithm(final long timeout, BenefitPolicy benefitPolicy, CuboidStats cuboidStats) {
+        if (timeout <= 0) {
+            this.timeoutMillis = Long.MAX_VALUE;
+        } else {
+            this.timeoutMillis = timeout;
+        }
+        this.cuboidStats = cuboidStats;
+        this.benefitPolicy = benefitPolicy;
+    }
+
+    @Override
+    public void cancel() {
+        cancelRequested.set(true);
+    }
+
+    /**
+     * Checks whether the algorithm has been canceled or timed out.
+     * 
+     */
+    protected boolean shouldCancel() {
+        if (canceled.get()) {
+            return true;
+        }
+        if (cancelRequested.get()) {
+            canceled.set(true);
+            cancelRequested.set(false);
+            logger.warn("Algorithm is canceled.");
+            return true;
+        }
+        final long currentTimeMillis = System.currentTimeMillis();
+        if (currentTimeMillis > timeoutMillis) {
+            canceled.set(true);
+            logger.warn("Algorithm exceeds time limit.");
+            return true;
+        }
+        return false;
+    }
+
+    public CuboidStats getCuboidStats() {
+        return cuboidStats;
+    }
+
+    public BenefitPolicy getBenefitPolicy() {
+        return benefitPolicy;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
new file mode 100755
index 0000000..6d0b654
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * Calculate the benefit based on Benefit Per Unit Space.
+ */
+public class BPUSCalculator implements BenefitPolicy {
+
+    private static Logger logger = LoggerFactory.getLogger(BPUSCalculator.class);
+
+    protected CuboidStats cuboidStats;
+    protected Map<Long, Long> cuboidAggCostMap;
+
+    public BPUSCalculator(CuboidStats cuboidStats) {
+        this.cuboidStats = cuboidStats;
+        this.cuboidAggCostMap = Maps.newHashMap();
+    }
+
+    @Override
+    public void initBeforeStart() {
+        cuboidAggCostMap.clear();
+        //Initialize stats for mandatory cuboids
+        for (Long cuboid : cuboidStats.getAllCuboidsForMandatory()) {
+            if (getCuboidCost(cuboid) != null) {
+                cuboidAggCostMap.put(cuboid, getCuboidCost(cuboid));
+            }
+        }
+        Set<Long> mandatoryCuboidSetWithStats = cuboidAggCostMap.keySet();
+        //Initialize stats for selection cuboids
+        long baseCuboidCost = getCuboidCost(cuboidStats.getBaseCuboid());
+        for (Long cuboid : cuboidStats.getAllCuboidsForSelection()) {
+            long leastCost = baseCuboidCost;
+            for (Long cuboidTarget : mandatoryCuboidSetWithStats) {
+                if ((cuboid | cuboidTarget) == cuboidTarget) {
+                    if (leastCost > cuboidAggCostMap.get(cuboidTarget)) {
+                        leastCost = cuboidAggCostMap.get(cuboidTarget);
+                    }
+                }
+            }
+            cuboidAggCostMap.put(cuboid, leastCost);
+        }
+    }
+
+    @Override
+    public CuboidBenefitModel.BenefitModel calculateBenefit(long cuboid, Set<Long> selected) {
+        double totalCostSaving = 0;
+        int benefitCount = 0;
+        for (Long descendant : cuboidStats.getAllDescendants(cuboid)) {
+            if (!selected.contains(descendant)) {
+                double costSaving = getCostSaving(descendant, cuboid);
+                if (costSaving > 0) {
+                    totalCostSaving += costSaving;
+                    benefitCount++;
+                }
+            }
+        }
+
+        double spaceCost = calculateSpaceCost(cuboid);
+        double benefitPerUnitSpace = totalCostSaving / spaceCost;
+        return new CuboidBenefitModel.BenefitModel(benefitPerUnitSpace, benefitCount);
+    }
+
+    @Override
+    public CuboidBenefitModel.BenefitModel calculateBenefitTotal(List<Long> cuboidsToAdd, Set<Long> selected) {
+        Set<Long> selectedInner = Sets.newHashSet(selected);
+        Map<Long, Long> cuboidAggCostMapSnapshot = Maps.newHashMap(cuboidAggCostMap);
+        for (Long cuboid : cuboidsToAdd) {
+            selectedInner.add(cuboid);
+            propagateAggregationCost(cuboid, selectedInner);
+        }
+        double totalCostSaving = 0;
+        int benefitCount = 0;
+        for (Long cuboid : cuboidAggCostMap.keySet()) {
+            if (cuboidAggCostMap.get(cuboid) < cuboidAggCostMapSnapshot.get(cuboid)) {
+                totalCostSaving += cuboidAggCostMapSnapshot.get(cuboid) - cuboidAggCostMap.get(cuboid);
+                benefitCount++;
+            }
+        }
+        cuboidAggCostMap = cuboidAggCostMapSnapshot;
+
+        double benefitPerUnitSpace = totalCostSaving;
+        return new CuboidBenefitModel.BenefitModel(benefitPerUnitSpace, benefitCount);
+    }
+
+    protected double getCostSaving(long descendant, long cuboid) {
+        long cuboidCost = getCuboidCost(cuboid);
+        long descendantAggCost = getCuboidAggregationCost(descendant);
+        return descendantAggCost - cuboidCost;
+    }
+
+    protected Long getCuboidCost(long cuboid) {
+        return cuboidStats.getCuboidCount(cuboid);
+    }
+
+    private long getCuboidAggregationCost(long cuboid) {
+        return cuboidAggCostMap.get(cuboid);
+    }
+
+    @Override
+    public boolean ifEfficient(CuboidBenefitModel best) {
+        if (best.getBenefit() < getMinBenefitRatio()) {
+            logger.info(String.format("The recommended cuboid %s doesn't meet minimum benifit ratio %f", best,
+                    getMinBenefitRatio()));
+            return false;
+        }
+        return true;
+    }
+
+    public double getMinBenefitRatio() {
+        return 0.01;
+    }
+
+    @Override
+    public void propagateAggregationCost(long cuboid, Set<Long> selected) {
+        long aggregationCost = getCuboidCost(cuboid);
+        Set<Long> childrenCuboids = cuboidStats.getAllDescendants(cuboid);
+        for (Long child : childrenCuboids) {
+            if (!selected.contains(child) && (aggregationCost < getCuboidAggregationCost(child))) {
+                cuboidAggCostMap.put(child, aggregationCost);
+            }
+        }
+    }
+
+    /**
+     * Return the space cost of building a cuboid.
+     *
+     */
+    public double calculateSpaceCost(long cuboid) {
+        return cuboidStats.getCuboidCount(cuboid);
+    }
+
+    @Override
+    public BenefitPolicy getInstance() {
+        BPUSCalculator bpusCalculator = new BPUSCalculator(this.cuboidStats);
+        bpusCalculator.cuboidAggCostMap.putAll(this.cuboidAggCostMap);
+        return bpusCalculator;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java
new file mode 100755
index 0000000..43bd3af
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+* Calculate the benefit of building the cuboid, the already selected cuboids will affect the benefits of the given cuboid
+* 
+*/
+public interface BenefitPolicy {
+
+    public BenefitPolicy getInstance();
+
+    public void initBeforeStart();
+
+    public CuboidBenefitModel.BenefitModel calculateBenefit(long cuboid, Set<Long> selected);
+
+    public CuboidBenefitModel.BenefitModel calculateBenefitTotal(List<Long> cuboidsToAdd, Set<Long> selected);
+
+    public boolean ifEfficient(CuboidBenefitModel best);
+
+    public void propagateAggregationCost(long cuboid, Set<Long> selected);
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java
new file mode 100755
index 0000000..8c9bbfa
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+public class CuboidBenefitModel {
+    private CuboidModel cuboidModel;
+    private BenefitModel benefitModel;
+
+    public CuboidBenefitModel(CuboidModel cuboidModel, BenefitModel benefitModel) {
+        this.cuboidModel = cuboidModel;
+        this.benefitModel = benefitModel;
+    }
+
+    public void reset(CuboidModel cuboidModel, BenefitModel benefitModel) {
+        this.cuboidModel = cuboidModel;
+        this.benefitModel = benefitModel;
+    }
+
+    public Long getCuboidId() {
+        return cuboidModel == null ? null : cuboidModel.getCuboidId();
+    }
+
+    public Double getBenefit() {
+        return benefitModel == null ? null : benefitModel.getBenefit();
+    }
+
+    @Override
+    public String toString() {
+        return "CuboidBenefitModel [cuboidModel=" + cuboidModel + ", benefitModel=" + benefitModel + "]";
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((cuboidModel == null) ? 0 : cuboidModel.hashCode());
+        result = prime * result + ((benefitModel == null) ? 0 : benefitModel.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        CuboidBenefitModel other = (CuboidBenefitModel) obj;
+        if (cuboidModel == null) {
+            if (other.cuboidModel != null)
+                return false;
+        } else if (!cuboidModel.equals(other.cuboidModel))
+            return false;
+        if (benefitModel == null) {
+            if (other.benefitModel != null)
+                return false;
+        } else if (!benefitModel.equals(other.benefitModel))
+            return false;
+        return true;
+    }
+
+    public static class CuboidModel {
+        private long cuboidId;
+
+        private long recordCount;
+        private double spaceSize;
+
+        private double hitProbability;
+        private long scanCount;
+
+        public CuboidModel(long cuboId, long recordCount, double spaceSize, double hitProbability, long scanCount) {
+            this.cuboidId = cuboId;
+            this.recordCount = recordCount;
+            this.spaceSize = spaceSize;
+            this.hitProbability = hitProbability;
+            this.scanCount = scanCount;
+        }
+
+        public long getCuboidId() {
+            return cuboidId;
+        }
+
+        public long getRecordCount() {
+            return recordCount;
+        }
+
+        public double getSpaceSize() {
+            return spaceSize;
+        }
+
+        public double getHitProbability() {
+            return hitProbability;
+        }
+
+        public long getScanCount() {
+            return scanCount;
+        }
+
+        @Override
+        public String toString() {
+            return "CuboidModel [cuboidId=" + cuboidId + ", recordCount=" + recordCount + ", spaceSize=" + spaceSize
+                    + ", hitProbability=" + hitProbability + ", scanCount=" + scanCount + "]";
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + (int) (cuboidId ^ (cuboidId >>> 32));
+            result = prime * result + (int) (recordCount ^ (recordCount >>> 32));
+            long temp;
+            temp = Double.doubleToLongBits(spaceSize);
+            result = prime * result + (int) (temp ^ (temp >>> 32));
+            temp = Double.doubleToLongBits(hitProbability);
+            result = prime * result + (int) (temp ^ (temp >>> 32));
+            result = prime * result + (int) (scanCount ^ (scanCount >>> 32));
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+
+            CuboidModel other = (CuboidModel) obj;
+            if (cuboidId != other.cuboidId)
+                return false;
+            if (recordCount != other.recordCount)
+                return false;
+            if (Double.doubleToLongBits(spaceSize) != Double.doubleToLongBits(other.spaceSize))
+                return false;
+            if (hitProbability != other.hitProbability)
+                return false;
+            if (scanCount != other.scanCount)
+                return false;
+            return true;
+        }
+    }
+
+    public static class BenefitModel {
+        private double benefit;
+        private int benefitCount;
+
+        public BenefitModel(double benefit, int benefitCount) {
+            this.benefit = benefit;
+            this.benefitCount = benefitCount;
+        }
+
+        public double getBenefit() {
+            return benefit;
+        }
+
+        public int getBenefitCount() {
+            return benefitCount;
+        }
+
+        @Override
+        public String toString() {
+            return "BenefitModel [benefit=" + benefit + ", benefitCount=" + benefitCount + "]";
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            long temp;
+            temp = Double.doubleToLongBits(benefit);
+            result = prime * result + (int) (temp ^ (temp >>> 32));
+            result = prime * result + benefitCount;
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            BenefitModel other = (BenefitModel) obj;
+            if (Double.doubleToLongBits(benefit) != Double.doubleToLongBits(other.benefit))
+                return false;
+            if (benefitCount != other.benefitCount)
+                return false;
+            return true;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java
new file mode 100755
index 0000000..b9f39ec
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+
+/**
+ * Algorithm to calculate the cuboid benifit and recommend cost-effective cuboid list based on the cube statistics.
+ */
+public interface CuboidRecommendAlgorithm {
+
+    /**
+     * Return a list of recommended cuboids for the building segment based on expansionRate
+     *
+     */
+    List<Long> recommend(double expansionRate);
+
+    /**
+     * Start the Algorithm running
+     * 
+     * @param maxSpaceLimit
+     * @return
+     */
+    List<Long> start(double maxSpaceLimit);
+
+    /**
+     * Cancel the Algorithm running
+     *
+     *  Users can call this method from another thread to can the Algorithm and return the result earlier
+     *
+     */
+    void cancel();
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
new file mode 100755
index 0000000..1775d5a
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CuboidStats {
+    private static final Logger logger = LoggerFactory.getLogger(CuboidStats.class);
+
+    public static class Builder {
+
+        private static final long THRESHOLD_ROLL_UP_FOR_MANDATORY = 1000L;
+
+        // Required parameters
+        private String key;
+        private Long baseCuboid;
+        private Map<Long, Long> statistics;
+        private Map<Long, Double> size;
+
+        // Optional parameters - initialized to default values
+        private Set<Long> mandatoryCuboids = null;
+        //// These two properties are for generating mandatory cuboids
+        private Map<Long, Map<Long, Long>> rollingUpCountSourceMap = null;
+        private Long rollUpThresholdForMandatory = null;
+
+        private Map<Long, Long> hitFrequencyMap = null;
+        private Map<Long, Map<Long, Long>> scanCountSourceMap = null;
+
+        public Builder(String key, Long baseCuboid, Map<Long, Long> statistics, Map<Long, Double> size) {
+            this.key = key;
+            this.baseCuboid = baseCuboid;
+            this.statistics = statistics;
+            this.size = size;
+        }
+
+        public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap) {
+            this.rollingUpCountSourceMap = rollingUpCountSourceMap;
+            this.rollUpThresholdForMandatory = THRESHOLD_ROLL_UP_FOR_MANDATORY;
+            return this;
+        }
+
+        public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap,
+                long rollUpThresholdForMandatory) {
+            this.rollingUpCountSourceMap = rollingUpCountSourceMap;
+            this.rollUpThresholdForMandatory = rollUpThresholdForMandatory;
+            return this;
+        }
+
+        public Builder setMandatoryCuboids(Set<Long> mandatoryCuboids) {
+            this.mandatoryCuboids = mandatoryCuboids;
+            return this;
+        }
+
+        public Builder setHitFrequencyMap(Map<Long, Long> hitFrequencyMap) {
+            this.hitFrequencyMap = hitFrequencyMap;
+            return this;
+        }
+
+        public Builder setScanCountSourceMap(Map<Long, Map<Long, Long>> scanCountSourceMap) {
+            this.scanCountSourceMap = scanCountSourceMap;
+            return this;
+        }
+
+        public CuboidStats build() {
+            Preconditions.checkNotNull(key, "key should not be null");
+            Preconditions.checkNotNull(baseCuboid, "baseCuboid should not be null");
+            Preconditions.checkNotNull(statistics, "statistics should not be null");
+            Preconditions.checkNotNull(size, "size should not be null");
+            Preconditions.checkNotNull(statistics.get(baseCuboid),
+                    "row count should exist for base cuboid " + baseCuboid);
+            Preconditions.checkState(statistics.keySet().equals(size.keySet()),
+                    "statistics & size should own the same key set");
+            if (mandatoryCuboids == null) {
+                mandatoryCuboids = Sets.newHashSet();
+            }
+            if (rollingUpCountSourceMap != null) {
+                mandatoryCuboids.addAll(CuboidStatsUtil.generateMandatoryCuboidSet(statistics, hitFrequencyMap,
+                        rollingUpCountSourceMap, rollUpThresholdForMandatory));
+            }
+
+            return new CuboidStats(key, baseCuboid, mandatoryCuboids, statistics, size, hitFrequencyMap,
+                    scanCountSourceMap);
+        }
+    }
+
+    private static final double WEIGHT_FOR_UN_QUERY = 0.2;
+
+    private String key;
+    private long baseCuboid;
+    private ImmutableSet<Long> mandatoryCuboidSet;
+    private ImmutableSet<Long> selectionCuboidSet;
+    private ImmutableMap<Long, Long> cuboidCountMap;
+    private ImmutableMap<Long, Double> cuboidSizeMap;
+    private ImmutableMap<Long, Double> cuboidHitProbabilityMap;
+    private ImmutableMap<Long, Long> cuboidScanCountMap;
+
+    private ImmutableMap<Long, Set<Long>> allDescendantsCache;
+
+    private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, Map<Long, Long> statistics,
+            Map<Long, Double> size, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
+
+        this.key = key;
+        this.baseCuboid = baseCuboidId;
+        /** Initial mandatory cuboids */
+        Set<Long> cuboidsForMandatory = Sets.newHashSet(mandatoryCuboids);
+        //Always add base cuboid.
+        if (!cuboidsForMandatory.contains(baseCuboid)) {
+            cuboidsForMandatory.add(baseCuboid);
+        }
+        logger.info("Mandatory cuboids: " + cuboidsForMandatory);
+
+        /** Initial selection cuboids */
+        Set<Long> cuboidsForSelection = Sets.newHashSet(statistics.keySet());
+        cuboidsForSelection.removeAll(cuboidsForMandatory);
+
+        //There's no overlap between mandatoryCuboidSet and selectionCuboidSet
+        this.mandatoryCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForMandatory).build();
+        this.selectionCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForSelection).build();
+        if (selectionCuboidSet.isEmpty()) {
+            logger.warn("The selection set should not be empty!!!");
+        }
+
+        /** Initialize row count for mandatory cuboids */
+        CuboidStatsUtil.complementRowCountForMandatoryCuboids(statistics, baseCuboid, mandatoryCuboidSet);
+
+        this.cuboidCountMap = ImmutableMap.<Long, Long> builder().putAll(statistics).build();
+        this.cuboidSizeMap = ImmutableMap.<Long, Double> builder().putAll(size).build();
+
+        /** Initialize the hit probability for each selection cuboid */
+        Map<Long, Double> tmpCuboidHitProbabilityMap = Maps.newHashMapWithExpectedSize(selectionCuboidSet.size());
+        if (hitFrequencyMap != null) {
+            long totalHitFrequency = 0L;
+            for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
+                if (selectionCuboidSet.contains(hitFrequency.getKey())) {
+                    totalHitFrequency += hitFrequency.getValue();
+                }
+            }
+
+            final double unitUncertainProb = WEIGHT_FOR_UN_QUERY / selectionCuboidSet.size();
+            for (Long cuboid : selectionCuboidSet) {
+                //Calculate hit probability for each cuboid
+                if (hitFrequencyMap.get(cuboid) != null) {
+                    tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb
+                            + (1 - WEIGHT_FOR_UN_QUERY) * hitFrequencyMap.get(cuboid) / totalHitFrequency);
+                } else {
+                    tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb);
+                }
+            }
+        } else {
+            for (Long cuboid : selectionCuboidSet) {
+                tmpCuboidHitProbabilityMap.put(cuboid, 1.0 / selectionCuboidSet.size());
+            }
+        }
+        this.cuboidHitProbabilityMap = ImmutableMap.<Long, Double> builder().putAll(tmpCuboidHitProbabilityMap).build();
+
+        /** Initialize the scan count when query for each selection cuboid + one base cuboid */
+        Map<Long, Long> tmpCuboidScanCountMap = Maps.newHashMapWithExpectedSize(1 + selectionCuboidSet.size());
+        tmpCuboidScanCountMap.put(baseCuboid, getExpScanCount(baseCuboid, statistics, scanCountSourceMap));
+        for (Long cuboid : selectionCuboidSet) {
+            tmpCuboidScanCountMap.put(cuboid, getExpScanCount(cuboid, statistics, scanCountSourceMap));
+        }
+        this.cuboidScanCountMap = ImmutableMap.<Long, Long> builder().putAll(tmpCuboidScanCountMap).build();
+
+        this.allDescendantsCache = ImmutableMap.<Long, Set<Long>> builder()
+                .putAll(CuboidStatsUtil.createAllDescendantsCache(statistics.keySet())).build();
+    }
+
+    private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
+            Map<Long, Map<Long, Long>> scanCountSourceMap) {
+        Preconditions.checkNotNull(statistics.get(sourceCuboid),
+                "The statistics for source cuboid " + sourceCuboid + " does not exist!!!");
+        if (scanCountSourceMap == null || scanCountSourceMap.get(sourceCuboid) == null
+                || scanCountSourceMap.get(sourceCuboid).size() <= 0) {
+            return statistics.get(sourceCuboid);
+        } else {
+            //TODO some improvement can be done by assigning weights based on distance between source cuboid and target cuboid
+            Map<Long, Long> scanCountTargetMap = scanCountSourceMap.get(sourceCuboid);
+            long totalEstScanCount = 0L;
+            for (Map.Entry<Long, Long> subEntry : scanCountTargetMap.entrySet()) {
+                long targetCuboid = subEntry.getKey();
+                Preconditions.checkNotNull(statistics.get(targetCuboid),
+                        "The statistics for target cuboid " + targetCuboid + " does not exist!!!");
+                // Consider the ratio of row count between source cuboid and target cuboid
+                totalEstScanCount += subEntry.getValue() * statistics.get(sourceCuboid) / statistics.get(targetCuboid);
+            }
+            return totalEstScanCount / scanCountTargetMap.size();
+        }
+    }
+
+    public Set<Long> getAllDescendants(long cuboid) {
+        Set<Long> allDescendants = Sets.newLinkedHashSet();
+        if (selectionCuboidSet.contains(cuboid)) {
+            return allDescendantsCache.get(cuboid);
+        }
+        return allDescendants;
+    }
+
+    public Set<Long> getAllCuboidsForSelection() {
+        return selectionCuboidSet;
+    }
+
+    public Set<Long> getAllCuboidsForMandatory() {
+        return mandatoryCuboidSet;
+    }
+
+    public Long getCuboidQueryCost(long cuboid) {
+        return cuboidScanCountMap.get(cuboid);
+    }
+
+    public Long getCuboidCount(long cuboid) {
+        return cuboidCountMap.get(cuboid);
+    }
+
+    public Double getCuboidSize(long cuboid) {
+        return cuboidSizeMap.get(cuboid);
+    }
+
+    public double getCuboidHitProbability(long cuboid) {
+        if (mandatoryCuboidSet.contains(cuboid)) {
+            return 1;
+        } else {
+            return cuboidHitProbabilityMap.get(cuboid) == null ? 0 : cuboidHitProbabilityMap.get(cuboid);
+        }
+    }
+
+    public Map<Long, Long> getStatistics() {
+        return cuboidCountMap;
+    }
+
+    public double getBaseCuboidSize() {
+        return getCuboidSize(baseCuboid);
+    }
+
+    public long getBaseCuboid() {
+        return baseCuboid;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public CuboidBenefitModel.CuboidModel getCuboidModel(long cuboid) {
+        return new CuboidBenefitModel.CuboidModel(cuboid, getCuboidCount(cuboid), getCuboidSize(cuboid),
+                getCuboidHitProbability(cuboid), getCuboidQueryCost(cuboid));
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
new file mode 100644
index 0000000..6d5bbe5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CuboidStatsUtil {
+
+    /**
+     * For generating mandatory cuboids,
+     * a cuboid is mandatory if the expectation of rolling up count exceeds a threshold
+     * */
+    public static Set<Long> generateMandatoryCuboidSet(Map<Long, Long> statistics, Map<Long, Long> hitFrequencyMap,
+            Map<Long, Map<Long, Long>> rollingUpCountSourceMap, final long rollUpThresholdForMandatory) {
+        Set<Long> mandatoryCuboidSet = Sets.newHashSet();
+        if (hitFrequencyMap == null || hitFrequencyMap.isEmpty() || rollingUpCountSourceMap == null
+                || rollingUpCountSourceMap.isEmpty()) {
+            return mandatoryCuboidSet;
+        }
+        long totalHitFrequency = 0L;
+        for (long hitFrequency : hitFrequencyMap.values()) {
+            totalHitFrequency += hitFrequency;
+        }
+
+        if (totalHitFrequency == 0) {
+            return mandatoryCuboidSet;
+        }
+
+        for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
+            long cuboid = hitFrequency.getKey();
+            if (statistics.get(cuboid) != null) {
+                continue;
+            }
+            if (rollingUpCountSourceMap.get(cuboid) == null || rollingUpCountSourceMap.get(cuboid).isEmpty()) {
+                continue;
+            }
+            long totalEstScanCount = 0L;
+            for (long estScanCount : rollingUpCountSourceMap.get(cuboid).values()) {
+                totalEstScanCount += estScanCount;
+            }
+            totalEstScanCount /= rollingUpCountSourceMap.get(cuboid).size();
+            if ((hitFrequency.getValue() * 1.0 / totalHitFrequency)
+                    * totalEstScanCount >= rollUpThresholdForMandatory) {
+                mandatoryCuboidSet.add(cuboid);
+            }
+        }
+        return mandatoryCuboidSet;
+    }
+
+    /**
+     * Complement row count for mandatory cuboids
+     * with its best parent's row count
+     * */
+    public static void complementRowCountForMandatoryCuboids(Map<Long, Long> statistics, long baseCuboid,
+            Set<Long> mandatoryCuboidSet) {
+        // Sort entries order by row count asc
+        SortedSet<Map.Entry<Long, Long>> sortedStatsSet = new TreeSet<Map.Entry<Long, Long>>(
+                new Comparator<Map.Entry<Long, Long>>() {
+                    public int compare(Map.Entry<Long, Long> o1, Map.Entry<Long, Long> o2) {
+                        return o1.getValue().compareTo(o2.getValue());
+                    }
+                });
+        sortedStatsSet.addAll(statistics.entrySet());
+        for (Long cuboid : mandatoryCuboidSet) {
+            if (statistics.get(cuboid) == null) {
+                // Get estimate row count for mandatory cuboid
+                long tmpRowCount = -1;
+                for (Map.Entry<Long, Long> entry : sortedStatsSet) {
+                    if (isDescendant(cuboid, entry.getKey())) {
+                        tmpRowCount = entry.getValue();
+                    }
+                }
+                statistics.put(cuboid, tmpRowCount < 0 ? statistics.get(baseCuboid) : tmpRowCount);
+            }
+        }
+    }
+
+    /** Using dynamic programming to use extra space to reduce repetitive computation*/
+    public static Map<Long, Set<Long>> createAllDescendantsCache(final Set<Long> cuboidSet) {
+        List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
+        Collections.sort(latticeCuboidList);
+
+        Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
+        Set<Long> preNoneDescendants = Sets.newHashSet();
+        for (int i = 0; i < latticeCuboidList.size(); i++) {
+            Long currentCuboid = latticeCuboidList.get(i);
+            Set<Long> descendants = Sets.newHashSet(currentCuboid);
+            Set<Long> curNoneDescendants = Sets.newHashSet();
+            if (i > 0) {
+                long preCuboid = latticeCuboidList.get(i - 1);
+                if (isDescendant(preCuboid, currentCuboid)) {
+                    descendants.addAll(allDescendantsCache.get(preCuboid));
+                } else {
+                    curNoneDescendants.add(preCuboid);
+                    for (long cuboidToCheck : allDescendantsCache.get(preCuboid)) {
+                        if (isDescendant(cuboidToCheck, currentCuboid)) {
+                            descendants.addAll(allDescendantsCache.get(cuboidToCheck));
+                        }
+                    }
+                }
+            }
+            for (long cuboidToCheck : preNoneDescendants) {
+                if (isDescendant(cuboidToCheck, currentCuboid)) {
+                    descendants.addAll(allDescendantsCache.get(cuboidToCheck));
+                } else {
+                    curNoneDescendants.add(cuboidToCheck);
+                }
+            }
+
+            allDescendantsCache.put(currentCuboid, descendants);
+            preNoneDescendants = curNoneDescendants;
+        }
+
+        return allDescendantsCache;
+    }
+
+    @VisibleForTesting
+    static Map<Long, Set<Long>> createAllDescendantsCache2(final Set<Long> cuboidSet) {
+        List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
+
+        Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
+        for (int i = 0; i < latticeCuboidList.size(); i++) {
+            Long currentCuboid = latticeCuboidList.get(i);
+            Set<Long> descendantSet = Sets.newHashSet(currentCuboid);
+            for (int j = 0; j < i; j++) {
+                Long checkCuboid = latticeCuboidList.get(j);
+                if (isDescendant(checkCuboid, currentCuboid)) {
+                    descendantSet.add(checkCuboid);
+                }
+            }
+            allDescendantsCache.put(currentCuboid, descendantSet);
+        }
+        return allDescendantsCache;
+    }
+
+    public static boolean isDescendant(long cuboidToCheck, long parentCuboid) {
+        return (cuboidToCheck & parentCuboid) == cuboidToCheck;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java
new file mode 100755
index 0000000..6d3ddc7
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Calculate the benefit based on Benefit Per Unit Space with query probability distribution.
+ */
+public class PBPUSCalculator extends BPUSCalculator {
+
+    public PBPUSCalculator(final CuboidStats cuboidStats) {
+        super(cuboidStats);
+    }
+
+    @Override
+    protected double getCostSaving(long descendant, long cuboid) {
+        return getCuboidHitProbability(descendant) * super.getCostSaving(descendant, cuboid);
+    }
+
+    protected double getCuboidHitProbability(long cuboid) {
+        return cuboidStats.getCuboidHitProbability(cuboid);
+    }
+
+    public double getMinBenefitRatio() {
+        Preconditions.checkArgument(cuboidStats.getAllCuboidsForSelection().size() > 0,
+                "The set of cuboids for selection is empty!!!");
+        return super.getMinBenefitRatio() / cuboidStats.getAllCuboidsForSelection().size();
+    }
+
+    @Override
+    public BenefitPolicy getInstance() {
+        PBPUSCalculator pbpusCalculator = new PBPUSCalculator(cuboidStats);
+        pbpusCalculator.cuboidAggCostMap.putAll(this.cuboidAggCostMap);
+        return pbpusCalculator;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java
new file mode 100755
index 0000000..c89bf49
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+/**
+ * Calculate the benefit based on Benefit Per Unit Space with query probability distribution and updated cost.
+ */
+public class SPBPUSCalculator extends PBPUSCalculator {
+
+    public SPBPUSCalculator(final CuboidStats cuboidStats) {
+        super(cuboidStats);
+    }
+
+    @Override
+    protected Long getCuboidCost(long cuboid) {
+        return cuboidStats.getCuboidQueryCost(cuboid);
+    }
+
+    @Override
+    public BenefitPolicy getInstance() {
+        SPBPUSCalculator spbpusCalculator = new SPBPUSCalculator(cuboidStats);
+        spbpusCalculator.cuboidAggCostMap.putAll(this.cuboidAggCostMap);
+        return spbpusCalculator;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java
new file mode 100755
index 0000000..02927c3
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler.CuboidCostComparator;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler.CuboidTree;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class AlgorithmTestBase {
+
+    public CuboidStats cuboidStats;
+
+    private Set<Long> mandatoryCuboids;
+
+    @Before
+    public void setUp() throws Exception {
+
+        mandatoryCuboids = Sets.newHashSet();
+        mandatoryCuboids.add(3000L);
+        mandatoryCuboids.add(1888L);
+        mandatoryCuboids.add(88L);
+        cuboidStats = new CuboidStats.Builder("test", 4095L, simulateCount(), simulateSpaceSize())
+                .setMandatoryCuboids(mandatoryCuboids).setHitFrequencyMap(simulateHitFrequency())
+                .setScanCountSourceMap(simulateScanCount()).build();
+    }
+
+    @After
+    public void after() throws Exception {
+    }
+
+    @Test
+    public void testMandatoryRowCountEstimation() {
+        for (long mandatoryCuboid : mandatoryCuboids) {
+            System.out.println("Cuboid id: " + mandatoryCuboid + "; Row Count: "
+                    + cuboidStats.getStatistics().get(mandatoryCuboid));
+        }
+    }
+
+    /** better if closer to 1, worse if closer to 0*/
+    public double getQueryCostRatio(CuboidStats cuboidStats, List<Long> recommendList) {
+        CuboidTree cuboidTree = CuboidTree.createFromCuboids(recommendList,
+                new CuboidCostComparator(cuboidStats.getStatistics()));
+        double queryCostBest = 0;
+        for (Long cuboidId : cuboidStats.getAllCuboidsForSelection()) {
+            if (cuboidStats.getCuboidQueryCost(cuboidId) != null) {
+                queryCostBest += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidCount(cuboidId);
+                //                queryCostBest += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidQueryCost(cuboidId);
+            }
+        }
+
+        double queryCost = 0;
+        for (Long cuboidId : cuboidStats.getAllCuboidsForSelection()) {
+            long matchCuboidId = cuboidTree.findBestMatchCuboid(cuboidId);
+            if (cuboidStats.getCuboidQueryCost(matchCuboidId) != null) {
+                queryCost += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidCount(matchCuboidId);
+                //                queryCost += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidQueryCost(matchCuboidId);
+            }
+        }
+
+        return queryCostBest / queryCost;
+    }
+
+    protected Map<Long, Long> simulateCount() {
+        Map<Long, Long> countMap = Maps.newHashMap();
+        BufferedReader br = null;
+
+        try {
+
+            String sCurrentLine;
+
+            br = new BufferedReader(new FileReader("src/test/resources/statistics.txt"));
+
+            while ((sCurrentLine = br.readLine()) != null) {
+                String[] statPair = sCurrentLine.split(" ");
+                countMap.put(Long.valueOf(statPair[0]), Long.valueOf(statPair[1]));
+            }
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            try {
+                if (br != null)
+                    br.close();
+            } catch (IOException ex) {
+                ex.printStackTrace();
+            }
+        }
+
+        return countMap;
+    }
+
+    protected Map<Long, Double> simulateSpaceSize() {
+        Map<Long, Double> sizeMap = Maps.newHashMap();
+        Map<Long, Long> countMap = simulateCount();
+        for (Map.Entry<Long, Long> entry : countMap.entrySet()) {
+            sizeMap.put(entry.getKey(), entry.getValue() * 1.0);
+        }
+        return sizeMap;
+    }
+
+    protected Map<Long, Long> simulateHitFrequency() {
+        Map<Long, Long> hitFrequencyMap = Maps.newHashMap();
+
+        hitFrequencyMap.put(4095L, 10L);
+        hitFrequencyMap.put(3849L, 15L);
+        hitFrequencyMap.put(3780L, 31L);
+
+        hitFrequencyMap.put(3459L, 16L);
+        hitFrequencyMap.put(3145L, 29L);
+
+        hitFrequencyMap.put(2861L, 21L);
+        hitFrequencyMap.put(2768L, 40L);
+
+        hitFrequencyMap.put(1528L, 10L);
+        hitFrequencyMap.put(1440L, 9L);
+        hitFrequencyMap.put(1152L, 21L);
+
+        hitFrequencyMap.put(256L, 23L);
+
+        hitFrequencyMap.put(128L, 7L);
+        hitFrequencyMap.put(272L, 8L);
+        hitFrequencyMap.put(288L, 10L);
+        hitFrequencyMap.put(384L, 2L);
+        hitFrequencyMap.put(320L, 3L);
+        hitFrequencyMap.put(432L, 5L);
+        hitFrequencyMap.put(258L, 8L);
+        hitFrequencyMap.put(336L, 10L);
+        hitFrequencyMap.put(274L, 22L);
+        hitFrequencyMap.put(488L, 41L);
+        hitFrequencyMap.put(352L, 10L);
+
+        hitFrequencyMap.put(16L, 1L);
+        hitFrequencyMap.put(32L, 5L);
+        hitFrequencyMap.put(34L, 1L);
+
+        hitFrequencyMap.put(2L, 21L);
+
+        return hitFrequencyMap;
+    }
+
+    protected Map<Long, Map<Long, Long>> simulateScanCount() {
+        Map<Long, Map<Long, Long>> scanCountMap = Maps.newLinkedHashMap();
+        scanCountMap.put(4094L, new HashMap<Long, Long>() {
+            {
+                put(4095L, 1833041L);
+            }
+        });
+        scanCountMap.put(3849L, new HashMap<Long, Long>() {
+            {
+                put(3849L, 276711L);
+            }
+        });
+        scanCountMap.put(3780L, new HashMap<Long, Long>() {
+            {
+                put(3780L, 129199L);
+            }
+        });
+        scanCountMap.put(3459L, new HashMap<Long, Long>() {
+            {
+                put(3459L, 168109L);
+            }
+        });
+        scanCountMap.put(3145L, new HashMap<Long, Long>() {
+            {
+                put(3145L, 299991L);
+            }
+        });
+        scanCountMap.put(2861L, new HashMap<Long, Long>() {
+            {
+                put(2861L, 2121L);
+            }
+        });
+        scanCountMap.put(2768L, new HashMap<Long, Long>() {
+            {
+                put(2768L, 40231L);
+            }
+        });
+        scanCountMap.put(256L, new HashMap<Long, Long>() {
+            {
+                put(256L, 1L);
+            }
+        });
+        scanCountMap.put(16L, new HashMap<Long, Long>() {
+            {
+                put(16L, 1L);
+            }
+        });
+        scanCountMap.put(32L, new HashMap<Long, Long>() {
+            {
+                put(32L, 2L);
+            }
+        });
+        scanCountMap.put(128L, new HashMap<Long, Long>() {
+            {
+                put(128L, 3L);
+            }
+        });
+        scanCountMap.put(272L, new HashMap<Long, Long>() {
+            {
+                put(272L, 2L);
+            }
+        });
+        scanCountMap.put(288L, new HashMap<Long, Long>() {
+            {
+                put(288L, 3L);
+            }
+        });
+        scanCountMap.put(2L, new HashMap<Long, Long>() {
+            {
+                put(2L, 1L);
+            }
+        });
+        scanCountMap.put(384L, new HashMap<Long, Long>() {
+            {
+                put(384L, 2L);
+            }
+        });
+        scanCountMap.put(320L, new HashMap<Long, Long>() {
+            {
+                put(320L, 3L);
+            }
+        });
+        scanCountMap.put(432L, new HashMap<Long, Long>() {
+            {
+                put(432L, 5L);
+            }
+        });
+        scanCountMap.put(1152L, new HashMap<Long, Long>() {
+            {
+                put(1152L, 21L);
+            }
+        });
+        scanCountMap.put(258L, new HashMap<Long, Long>() {
+            {
+                put(258L, 2L);
+            }
+        });
+        scanCountMap.put(1440L, new HashMap<Long, Long>() {
+            {
+                put(1440L, 9L);
+            }
+        });
+        scanCountMap.put(336L, new HashMap<Long, Long>() {
+            {
+                put(336L, 2L);
+            }
+        });
+        scanCountMap.put(336L, new HashMap<Long, Long>() {
+            {
+                put(336L, 2L);
+            }
+        });
+        scanCountMap.put(274L, new HashMap<Long, Long>() {
+            {
+                put(274L, 1L);
+            }
+        });
+        scanCountMap.put(488L, new HashMap<Long, Long>() {
+            {
+                put(488L, 16L);
+            }
+        });
+        scanCountMap.put(352L, new HashMap<Long, Long>() {
+            {
+                put(352L, 3L);
+            }
+        });
+        scanCountMap.put(1528L, new HashMap<Long, Long>() {
+            {
+                put(1528L, 21L);
+            }
+        });
+        scanCountMap.put(34L, new HashMap<Long, Long>() {
+            {
+                put(34L, 1L);
+            }
+        });
+
+        return scanCountMap;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e45864/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
new file mode 100644
index 0000000..8e809cf
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CuboidStatsUtilTest {
+
+    /**
+     *                 (11111111)
+     *               /     |      \
+     *      10011111   (11101111)  00110010
+     *           \         |         /
+     *             \   11000111     /
+     *               \     |       /
+     *                 00000110   /
+     *              /      |     /
+     *         00000100  00000010
+     * */
+    private Set<Long> generateCuboidSet() {
+        return Sets.newHashSet(255L, 159L, 239L, 50L, 199L, 6L, 4L, 2L);
+    }
+
+    private Map<Long, Long> simulateCount() {
+        Map<Long, Long> countMap = Maps.newHashMap();
+
+        countMap.put(255L, 10000L);
+        countMap.put(159L, 10000L);
+        countMap.put(50L, 10000L);
+        countMap.put(199L, 10000L);
+        countMap.put(6L, 10000L);
+        countMap.put(4L, 10000L);
+        countMap.put(2L, 10000L);
+
+        return countMap;
+    }
+
+    private Map<Long, Long> simulateHitFrequency() {
+        Map<Long, Long> hitFrequencyMap = Maps.newHashMap();
+
+        long totalHitFrequency = 10000L;
+
+        hitFrequencyMap.put(239L, (long) (totalHitFrequency * 0.5));
+        hitFrequencyMap.put(50L, (long) (totalHitFrequency * 0.2));
+        hitFrequencyMap.put(2L, (long) (totalHitFrequency * 0.25));
+        hitFrequencyMap.put(178L, (long) (totalHitFrequency * 0.05));
+
+        return hitFrequencyMap;
+    }
+
+    private Map<Long, Map<Long, Long>> simulateRollingUpCount() {
+        Map<Long, Map<Long, Long>> rollingUpCountMap = Maps.newLinkedHashMap();
+
+        rollingUpCountMap.put(239L, new HashMap<Long, Long>() {
+            {
+                put(255L, 4000L);
+            }
+        });
+
+        rollingUpCountMap.put(178L, new HashMap<Long, Long>() {
+            {
+                put(255L, 5000L);
+            }
+        });
+
+        return rollingUpCountMap;
+    }
+
+    @Test
+    public void isDescendantTest() {
+        Assert.assertTrue(CuboidStatsUtil.isDescendant(6L, 239L));
+        Assert.assertTrue(!CuboidStatsUtil.isDescendant(4L, 50L));
+    }
+
+    @Test
+    public void generateMandatoryCuboidSetTest() {
+        Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(simulateCount(),
+                simulateHitFrequency(), simulateRollingUpCount(), 1000L);
+        Assert.assertTrue(mandatoryCuboidSet.contains(239L));
+        Assert.assertTrue(!mandatoryCuboidSet.contains(178L));
+    }
+
+    @Test
+    public void complementRowCountForMandatoryCuboidsTest() {
+        Map<Long, Long> countMap = simulateCount();
+        Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(countMap, simulateHitFrequency(),
+                simulateRollingUpCount(), 1000L);
+        for (long mandatoryCuboid : mandatoryCuboidSet) {
+            Assert.assertNull(countMap.get(mandatoryCuboid));
+        }
+        CuboidStatsUtil.complementRowCountForMandatoryCuboids(countMap, 255L, mandatoryCuboidSet);
+        for (long mandatoryCuboid : mandatoryCuboidSet) {
+            Assert.assertNotNull(countMap.get(mandatoryCuboid));
+        }
+        Assert.assertTrue(countMap.get(239L) == 10000L);
+    }
+
+    @Test
+    public void createAllDescendantsCacheTest() {
+        Set<Long> cuboidSet = generateCuboidSet();
+        Map<Long, Set<Long>> allDescendantsCache = CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
+
+        Assert.assertTrue(allDescendantsCache.get(255L).containsAll(cuboidSet));
+
+        Assert.assertTrue(allDescendantsCache.get(239L).size() == 5);
+
+        Assert.assertTrue(allDescendantsCache.get(50L).containsAll(Sets.newHashSet(50L, 2L)));
+        Assert.assertTrue(!allDescendantsCache.get(50L).contains(4L));
+    }
+
+    private Set<Long> generateMassCuboidSet() {
+        Set<Long> cuboidSet = Sets.newHashSet();
+        long maxCuboid = (1L << 16);
+        for (long i = 0; i < maxCuboid; i++) {
+            cuboidSet.add(i);
+        }
+        return cuboidSet;
+    }
+
+    @Test
+    public void createAllDescendantsCacheStressTest() {
+        Stopwatch sw = new Stopwatch();
+        sw.start();
+        Set<Long> cuboidSet = generateMassCuboidSet();
+        System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS));
+        sw.reset();
+        sw.start();
+        CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
+        System.out.println("Time elapsed for creating descendants cache: " + sw.elapsed(TimeUnit.MILLISECONDS));
+        sw.reset();
+        sw.start();
+        CuboidStatsUtil.createAllDescendantsCache2(cuboidSet);
+        System.out.println("Time elapsed for creating descendants cache2: " + sw.elapsed(TimeUnit.MILLISECONDS));
+    }
+}


[10/22] kylin git commit: APACHE-KYLIN-2802: Enable cube planner phase one

Posted by nj...@apache.org.
APACHE-KYLIN-2802: Enable cube planner phase one


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed0cc11f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed0cc11f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed0cc11f

Branch: refs/heads/yaho-cube-planner
Commit: ed0cc11f306de32fc3feccb9b64aff31e55ecffd
Parents: b3c4c56
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 17:58:37 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:39 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |  27 +++
 .../org/apache/kylin/common/util/JsonUtil.java  |   4 +
 .../org/apache/kylin/cube/CubeInstance.java     |  19 ++-
 .../java/org/apache/kylin/cube/CubeManager.java |   4 +
 .../java/org/apache/kylin/cube/CubeUpdate.java  |  11 ++
 .../cuboid/algorithm/CuboidRecommender.java     | 163 +++++++++++++++++++
 .../org/apache/kylin/cube/CubeInstanceTest.java |  57 +++++++
 .../src/test/resources/learn_kylin_cube2.json   |  84 ++++++++++
 .../kylin/engine/mr/common/CubeStatsReader.java |   4 +
 .../engine/mr/common/CuboidRecommenderUtil.java | 106 ++++++++++++
 .../kylin/engine/mr/common/CuboidStatsUtil.java | 135 +++++++++++++++
 .../mr/common/StatisticsDecisionUtil.java       | 110 +++++++++++++
 .../engine/mr/steps/SaveStatisticsStep.java     |  55 +------
 13 files changed, 722 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6feb51a..d66d7ce 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1126,6 +1126,33 @@ abstract public class KylinConfigBase implements Serializable {
     }
 
     // ============================================================================
+    // Cube Planner
+    // ============================================================================
+    public boolean isCubePlannerEnabledForExistingCube() {
+        return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false"));
+    }
+
+    public double getCubePlannerExpansionRateThreshold() {
+        return Double.parseDouble(getOptional("kylin.cube.cubeplanner.expansion-threshold", "15.0"));
+    }
+
+    public int getCubePlannerRecommendCuboidCacheMaxSize() {
+        return Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size", "200"));
+    }
+
+    public long getCubePlannerMandatoryRollUpThreshold() {
+        return Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold", "1000"));
+    }
+
+    public int getCubePlannerAgreedyAlgorithmAutoThreshold() {
+        return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-greedy", "10"));
+    }
+
+    public int getCubePlannerGeneticAlgorithmAutoThreshold() {
+        return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-genetic", "23"));
+    }
+
+    // ============================================================================
     // Metrics
     // ============================================================================
     public boolean isKylinMetricsMonitorEnabled() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index f874b16..c30a1f2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -92,6 +92,10 @@ public class JsonUtil {
         return typeMapper.readValue(src, valueType);
     }
 
+    public static <T> T readValue(String content, TypeReference<T> valueTypeRef) throws IOException {
+        return mapper.readValue(content, valueTypeRef);
+    }
+
     public static void writeValueIndent(OutputStream out, Object value)
             throws IOException, JsonGenerationException, JsonMappingException {
         indentMapper.writeValue(out, value);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 3f8bbbe..d217dcd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -22,8 +22,6 @@ import static org.apache.kylin.cube.cuboid.CuboidModeEnum.CURRENT;
 import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -59,6 +57,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonManagedReference;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.base.Objects;
 import com.google.common.collect.Lists;
 
@@ -359,21 +358,23 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         }
     }
 
-    public HashMap<Long, Long> getCuboids() {
+    public Map<Long, Long> getCuboids() {
         if (cuboidBytes == null)
             return null;
         byte[] uncompressed;
         try {
             uncompressed = CompressionUtils.decompress(cuboidBytes);
             String str = new String(uncompressed, "UTF-8");
-            HashMap<Long, Long> cuboids = JsonUtil.readValue(str, HashMap.class);
+            TypeReference<Map<Long, Long>> typeRef = new TypeReference<Map<Long, Long>>() {
+            };
+            Map<Long, Long> cuboids = JsonUtil.readValue(str, typeRef);
             return cuboids.isEmpty() ? null : cuboids;
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
-    public void setCuboids(HashMap<Long, Long> cuboids) {
+    public void setCuboids(Map<Long, Long> cuboids) {
         if (cuboids == null)
             return;
         if (cuboids.isEmpty()) {
@@ -390,21 +391,23 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         }
     }
 
-    public HashSet<Long> getCuboidsRecommend() {
+    public Set<Long> getCuboidsRecommend() {
         if (cuboidBytesRecommend == null)
             return null;
         byte[] uncompressed;
         try {
             uncompressed = CompressionUtils.decompress(cuboidBytesRecommend);
             String str = new String(uncompressed, "UTF-8");
-            HashSet<Long> cuboids = JsonUtil.readValue(str, HashSet.class);
+            TypeReference<Set<Long>> typeRef = new TypeReference<Set<Long>>() {
+            };
+            Set<Long> cuboids = JsonUtil.readValue(str, typeRef);
             return cuboids.isEmpty() ? null : cuboids;
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
-    public void setCuboidsRecommend(HashSet<Long> cuboids) {
+    public void setCuboidsRecommend(Set<Long> cuboids) {
         if (cuboids == null)
             return;
         if (cuboids.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 67c1562..a679dde 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -421,6 +421,10 @@ public class CubeManager implements IRealizationProvider {
             cube.setCost(update.getCost());
         }
 
+        if(update.getCuboids() != null){
+            cube.setCuboids(update.getCuboids());
+        }
+
         try {
             getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
         } catch (IllegalStateException ise) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
index dffaa48..fae20dc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.cube;
 
+import java.util.Map;
+
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 
 /**
@@ -31,6 +33,7 @@ public class CubeUpdate {
     private RealizationStatusEnum status;
     private String owner;
     private int cost = -1;
+    private Map<Long, Long> cuboids = null;
 
     public CubeUpdate(CubeInstance cubeInstance) {
         this.cubeInstance = cubeInstance;
@@ -98,4 +101,12 @@ public class CubeUpdate {
         this.cost = cost;
         return this;
     }
+
+    public Map<Long, Long> getCuboids() {
+        return cuboids;
+    }
+
+    public void setCuboids(Map<Long, Long> cuboids) {
+        this.cuboids = cuboids;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
new file mode 100644
index 0000000..43b2318
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+import org.apache.kylin.cube.cuboid.algorithm.greedy.GreedyAlgorithm;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.Maps;
+
+public class CuboidRecommender {
+    private static final Logger logger = LoggerFactory.getLogger(CuboidRecommender.class);
+
+    private static Cache<String, Map<Long, Long>> cuboidRecommendCache = CacheBuilder.newBuilder()
+            .removalListener(new RemovalListener<String, Map<Long, Long>>() {
+                @Override
+                public void onRemoval(RemovalNotification<String, Map<Long, Long>> notification) {
+                    logger.info("Dict with resource path " + notification.getKey() + " is removed due to "
+                            + notification.getCause());
+                }
+            }).maximumSize(KylinConfig.getInstanceFromEnv().getCubePlannerRecommendCuboidCacheMaxSize())
+            .expireAfterWrite(1, TimeUnit.DAYS).build();
+
+    private class CuboidRecommenderSyncListener extends Broadcaster.Listener {
+        @Override
+        public void onClearAll(Broadcaster broadcaster) throws IOException {
+            cuboidRecommendCache.invalidateAll();
+        }
+
+        @Override
+        public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey)
+                throws IOException {
+            cuboidRecommendCache.invalidate(cacheKey);
+        }
+    }
+
+    public CuboidRecommender() {
+        Broadcaster.getInstance(KylinConfig.getInstanceFromEnv()).registerListener(new CuboidRecommenderSyncListener(),
+                "cube");
+    }
+
+    private static CuboidRecommender instance = new CuboidRecommender();
+
+    public static CuboidRecommender getInstance() {
+        return instance;
+    }
+
+    public Map<Long, Long> getRecommendCuboidList(final CuboidStats cuboidStats, final KylinConfig kylinConfig) {
+        if (cuboidStats == null) {
+            return null;
+        }
+        final String key = cuboidStats.getKey();
+        Map<Long, Long> results = cuboidRecommendCache.getIfPresent(key);
+        if (results == null) {
+            try {
+                results = cuboidRecommendCache.get(key, new Callable<Map<Long, Long>>() {
+                    @Override
+                    public Map<Long, Long> call() throws Exception {
+                        // recommending flag
+                        Map<Long, Long> emptyMap = Maps.newHashMap();
+                        cuboidRecommendCache.put(key, emptyMap);
+                        try {
+                            Map<Long, Long> recommendCuboid = getRecommendCuboidList(cuboidStats, kylinConfig,
+                                    true);
+
+                            if (recommendCuboid != null) {
+                                logger.info("Add recommend cuboids for " + key + " to cache");
+                                cuboidRecommendCache.put(key, recommendCuboid);
+                            }
+
+                            return recommendCuboid;
+                        } catch (Exception e) {
+                            cuboidRecommendCache.invalidate(key);
+                            logger.error("Failed to get recommend cuboids for " + key + " in cache", e);
+                            throw e;
+                        }
+                    }
+                });
+            } catch (ExecutionException e) {
+                logger.error("Failed to get recommend cuboids for " + key);
+            }
+        }
+        return results;
+    }
+
+    public Map<Long, Long> getRecommendCuboidList(CuboidStats cuboidStats, KylinConfig kylinConf,
+            boolean ifForceRecommend) {
+        long Threshold1 = 1L << kylinConf.getCubePlannerAgreedyAlgorithmAutoThreshold();
+        long Threshold2 = 1L << kylinConf.getCubePlannerGeneticAlgorithmAutoThreshold();
+        if (Threshold1 >= Threshold2) {
+            logger.error("Invalid Cube Planner Algorithm configuration");
+            return null;
+        }
+
+        int allCuboidCount = cuboidStats.getAllCuboidsForMandatory().size()
+                + cuboidStats.getAllCuboidsForSelection().size();
+
+        BenefitPolicy benefitPolicy = new PBPUSCalculator(cuboidStats);
+        CuboidRecommendAlgorithm algorithm = null;
+
+        if (allCuboidCount <= Threshold2) {
+            algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+        } else {
+            algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+        }
+
+        long startTime = System.currentTimeMillis();
+        logger.info("Cube Planner Algorithm started at " + startTime);
+        List<Long> recommendCuboidList = algorithm.recommend(kylinConf.getCubePlannerExpansionRateThreshold());
+        logger.info("Cube Planner Algorithm ended at " + (System.currentTimeMillis() - startTime));
+
+        if (recommendCuboidList.size() < allCuboidCount) {
+            logger.info("Cube Planner Algorithm chooses " + recommendCuboidList.size()
+                    + " most effective cuboids to build among of all " + allCuboidCount + " cuboids.");
+        }
+
+        Map<Long, Long> recommendCuboidsWithStats = Maps.newLinkedHashMap();
+        for (Long cuboid : recommendCuboidList) {
+            if (cuboid.equals(cuboidStats.getBaseCuboid())) {
+                recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
+            } else if (cuboidStats.getAllCuboidsForSelection().contains(cuboid)) {
+                recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
+            } else {
+                recommendCuboidsWithStats.put(cuboid, -1L);
+            }
+        }
+
+        if (!ifForceRecommend && allCuboidCount <= Threshold1) {
+            return null;
+        }
+        return recommendCuboidsWithStats;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
new file mode 100644
index 0000000..0af5f98
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Map;
+
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class CubeInstanceTest {
+
+    private CubeInstance cubeInstance;
+
+    @Before
+    public void setUp() throws Exception {
+        InputStream fileInputStream = new FileInputStream("src/test/resources/learn_kylin_cube2.json");
+        JsonSerializer<CubeInstance> jsonSerializer = new JsonSerializer<>(CubeInstance.class);
+        cubeInstance = jsonSerializer.deserialize(new DataInputStream(fileInputStream));
+    }
+
+    @Test
+    public void getTreeCuboidSchedulerTest() {
+        Map<Long, Long> cuboidsWithRowCnt = cubeInstance.getCuboids();
+        TreeCuboidScheduler.CuboidTree.createFromCuboids(Lists.newArrayList(cuboidsWithRowCnt.keySet()),
+                new TreeCuboidScheduler.CuboidCostComparator(cuboidsWithRowCnt));
+    }
+
+    public void printMap(Map<Long, Long> map) {
+        System.out.println("size: " + map.size());
+        for (Map.Entry<Long, Long> entry : map.entrySet()) {
+            System.out.println(entry.getKey() + ":" + entry.getValue());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/core-cube/src/test/resources/learn_kylin_cube2.json
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/learn_kylin_cube2.json b/core-cube/src/test/resources/learn_kylin_cube2.json
new file mode 100644
index 0000000..1f50902
--- /dev/null
+++ b/core-cube/src/test/resources/learn_kylin_cube2.json
@@ -0,0 +1,84 @@
+{
+  "uuid" : "bd66b324-4b0d-4fc7-a2ef-2b9ddced160d",
+  "last_modified" : 1503388415029,
+  "version" : "2.1.0",
+  "name" : "learn_kylin_cube2",
+  "owner" : "ADMIN",
+  "descriptor" : "learn_kylin_cube2",
+  "cost" : 50,
+  "status" : "READY",
+  "segments" : [ {
+    "uuid" : "4d017947-1a63-47a9-b31d-356219612cf6",
+    "name" : "20100101000000_20150101000000",
+    "storage_location_identifier" : "KYLIN_FKF6CWSIZC",
+    "date_range_start" : 1262304000000,
+    "date_range_end" : 1420070400000,
+    "source_offset_start" : 0,
+    "source_offset_end" : 0,
+    "status" : "READY",
+    "size_kb" : 22153,
+    "input_records" : 10000,
+    "input_records_size" : 984072,
+    "last_build_time" : 1503388048037,
+    "last_build_job_id" : "06c5bfd9-0443-4c28-810b-bcbd37fde458",
+    "create_time_utc" : 1503387574466,
+    "cuboid_shard_nums" : { },
+    "total_shards" : 1,
+    "blackout_cuboids" : [ ],
+    "binary_signature" : null,
+    "dictionaries" : {
+      "KYLIN_SALES.LSTG_SITE_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/SITE_ID/d1dbea6a-4929-43ef-bf2a-cf356d9de9eb.dict",
+      "KYLIN_SALES.LEAF_CATEG_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/LEAF_CATEG_ID/60f6c00a-2137-4d6f-90e6-5433d30e326d.dict",
+      "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/META_CATEG_NAME/e375c3b2-bebf-4f9a-95f3-abb76a7bdc80.dict",
+      "KYLIN_SALES.SLR_SEGMENT_CD" : "/dict/DEFAULT.KYLIN_SALES/SLR_SEGMENT_CD/ddb1659c-85cc-4d4f-8aae-40c6618f61cc.dict",
+      "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7bbcdb05-4cf9-4d31-942c-828c5bbcaab8.dict",
+      "KYLIN_SALES.LSTG_FORMAT_NAME" : "/dict/DEFAULT.KYLIN_SALES/LSTG_FORMAT_NAME/35669f98-38f2-4e67-9547-1b0b446a1675.dict",
+      "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/a83a92e5-7711-4a34-bff8-ab3da625e996.dict"
+    },
+    "snapshots" : {
+      "DEFAULT.KYLIN_CATEGORY_GROUPINGS" : "/table_snapshot/DEFAULT.KYLIN_CATEGORY_GROUPINGS/90b8251c-96c9-45a6-a4db-9fe49e8a0581.snapshot",
+      "DEFAULT.KYLIN_CAL_DT" : "/table_snapshot/DEFAULT.KYLIN_CAL_DT/35abb8cf-4e70-428e-adef-457e5a085aa0.snapshot"
+    },
+    "rowkey_stats" : [ [ "KYLIN_SALES.LSTG_FORMAT_NAME", 5, 1 ], [ "KYLIN_SALES.SLR_SEGMENT_CD", 8, 1 ], [ "KYLIN_SALES.LEAF_CATEG_ID", 134, 1 ], [ "KYLIN_SALES.LSTG_SITE_ID", 8, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", 44, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 94, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 127, 1 ] ]
+  }, {
+    "uuid" : "4cb8c3e7-9624-44c4-b812-041cb5604e50",
+    "name" : "20100101000000_20150101000000",
+    "storage_location_identifier" : "KYLIN_VEXWSEB5Y7",
+    "date_range_start" : 1262304000000,
+    "date_range_end" : 1420070400000,
+    "source_offset_start" : 0,
+    "source_offset_end" : 0,
+    "status" : "NEW",
+    "size_kb" : 0,
+    "input_records" : 0,
+    "input_records_size" : 0,
+    "last_build_time" : 0,
+    "last_build_job_id" : null,
+    "create_time_utc" : 1503388305176,
+    "cuboid_shard_nums" : { },
+    "total_shards" : 0,
+    "blackout_cuboids" : [ ],
+    "binary_signature" : null,
+    "dictionaries" : {
+      "KYLIN_SALES.LSTG_SITE_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/SITE_ID/d1dbea6a-4929-43ef-bf2a-cf356d9de9eb.dict",
+      "KYLIN_SALES.LEAF_CATEG_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/LEAF_CATEG_ID/60f6c00a-2137-4d6f-90e6-5433d30e326d.dict",
+      "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/META_CATEG_NAME/e375c3b2-bebf-4f9a-95f3-abb76a7bdc80.dict",
+      "KYLIN_SALES.SLR_SEGMENT_CD" : "/dict/DEFAULT.KYLIN_SALES/SLR_SEGMENT_CD/ddb1659c-85cc-4d4f-8aae-40c6618f61cc.dict",
+      "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7bbcdb05-4cf9-4d31-942c-828c5bbcaab8.dict",
+      "KYLIN_SALES.LSTG_FORMAT_NAME" : "/dict/DEFAULT.KYLIN_SALES/LSTG_FORMAT_NAME/35669f98-38f2-4e67-9547-1b0b446a1675.dict",
+      "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/a83a92e5-7711-4a34-bff8-ab3da625e996.dict"
+    },
+    "snapshots" : {
+      "DEFAULT.KYLIN_CATEGORY_GROUPINGS" : "/table_snapshot/DEFAULT.KYLIN_CATEGORY_GROUPINGS/90b8251c-96c9-45a6-a4db-9fe49e8a0581.snapshot",
+      "DEFAULT.KYLIN_CAL_DT" : "/table_snapshot/DEFAULT.KYLIN_CAL_DT/35abb8cf-4e70-428e-adef-457e5a085aa0.snapshot"
+    },
+    "rowkey_stats" : [ [ "KYLIN_SALES.LSTG_FORMAT_NAME", 5, 1 ], [ "KYLIN_SALES.SLR_SEGMENT_CD", 8, 1 ], [ "KYLIN_SALES.LEAF_CATEG_ID", 134, 1 ], [ "KYLIN_SALES.LSTG_SITE_ID", 8, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", 44, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 94, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 127, 1 ] ]
+  } ],
+  "create_time_utc" : 1502959568985,
+  "cuboidBytes" : "eAEdkTmWBTEIA+/iuAM24+Vq8+buv+TINEhCqP9GzDnuOebf6Bp3fmOPu76RMe7m8XG96huL76BRNm4yPz1uGS/0PhSdIG3lN9wQcg09llB9VEo31UyoPg2RMNQqDiQoxdOgwkEJlFrowiCEMzemCDMu8LnRXeBiaisG3yuJ2dm05uG2nG8p454yigDlKsgyYjqOzjPkBizNmSUyW6tc15hWSTgo3MGnp+JAYqd6R8zc2IyX3So5aJC1zgMAPTYpQwEdE9adgDkEqYXZ7hCJhYktJ1t/4glhb9NCEsXzg0kldceG0YtAXb/p5NK4KPdTLsVmkf8/QfNdrA==",
+  "cuboid_bytes_recommend" : null,
+  "last_optimized" : 0,
+  "size_kb" : 22153,
+  "input_records_count" : 10000,
+  "input_records_size" : 984072
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 417697d..8f1da6e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -140,6 +140,10 @@ public class CubeStatsReader {
         return tempFile;
     }
 
+    public Map<Long, HLLCounter> getCuboidRowEstimatesHLLOrigin() {
+        return this.cuboidRowEstimatesHLL;
+    }
+
     public Map<Long, Long> getCuboidRowEstimatesHLL() {
         return getCuboidRowCountMapFromSampling(cuboidRowEstimatesHLL, samplingPercentage);
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
new file mode 100644
index 0000000..a0b20fa
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidRecommender;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CuboidRecommenderUtil {
+
+    private static final Logger logger = LoggerFactory.getLogger(CuboidRecommenderUtil.class);
+
+    /** Trigger cube planner phase one */
+    public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment) throws IOException {
+        if (segment == null) {
+            return null;
+        }
+
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, segment.getConfig());
+        if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+                || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+            logger.info("Cuboid Statistics is not enabled.");
+            return null;
+        }
+        long baseCuboid = segment.getCuboidScheduler().getBaseCuboidId();
+        if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
+                || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
+            logger.info("Base cuboid count in cuboid statistics is 0.");
+            return null;
+        }
+
+        String key = segment.getCubeInstance().getName();
+        CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
+                cubeStatsReader.getCuboidSizeMap()).build();
+        return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(), false);
+    }
+
+    /** Trigger cube planner phase two for optimization */
+    public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+            Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+
+        Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsUtil
+                .readCuboidStatsAndSizeFromCube(cube.getCuboidScheduler().getAllCuboidIds(), cube);
+
+        String key = cube.getName();
+        long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
+        CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(), statsPair.getSecond())
+                .setHitFrequencyMap(hitFrequencyMap).setRollingUpCountSourceMap(rollingUpCountSourceMap,
+                        cube.getConfig().getCubePlannerMandatoryRollUpThreshold())
+                .build();
+        return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, cube.getConfig());
+    }
+
+    /** For future segment level recommend */
+    public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
+            Map<Long, Map<Long, Long>> rollingUpCountSourceMap, boolean ifForceRecommend) throws IOException {
+        if (segment == null) {
+            return null;
+        }
+
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, segment.getConfig());
+        if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+                || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+            logger.info("Cuboid Statistics is not enabled.");
+            return null;
+        }
+        long baseCuboid = segment.getCuboidScheduler().getBaseCuboidId();
+        if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
+                || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
+            logger.info("Base cuboid count in cuboid statistics is 0.");
+            return null;
+        }
+
+        String key = segment.getCubeInstance().getName() + "-" + segment.getName();
+        CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
+                cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
+                        .setRollingUpCountSourceMap(rollingUpCountSourceMap,
+                                segment.getConfig().getCubePlannerMandatoryRollUpThreshold())
+                        .build();
+        return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
+                ifForceRecommend);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
new file mode 100644
index 0000000..5bb4179
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class CuboidStatsUtil {
+
+    private static final Logger logger = LoggerFactory.getLogger(CuboidStatsUtil.class);
+
+    public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance)
+            throws IOException {
+        Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+        return statisticsMerged.isEmpty() ? null : statisticsMerged;
+    }
+
+    public static Pair<Map<Long, Long>, Map<Long, Double>> readCuboidStatsAndSizeFromCube(Set<Long> cuboidIds,
+            CubeInstance cube) throws IOException {
+        Preconditions.checkNotNull(cuboidIds, "The cuboid set can not be null");
+        Preconditions.checkNotNull(cube, "The cube instance can not be null");
+
+        List<CubeSegment> segmentList = cube.getSegments(SegmentStatusEnum.READY);
+        Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        Map<Long, Double> sizeMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged, sizeMerged);
+        return new Pair<>(statisticsMerged, sizeMerged);
+    }
+
+    public static Map<Long, Long> readCuboidStatsFromSegments(Set<Long> cuboidIds, List<CubeSegment> segmentList)
+            throws IOException {
+        Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged,
+                Maps.<Long, Double> newHashMapWithExpectedSize(cuboidIds.size()));
+        return statisticsMerged.isEmpty() ? null : statisticsMerged;
+    }
+
+    private static void readCuboidStatsFromSegments(Set<Long> cuboidSet, List<CubeSegment> segmentList,
+            final Map<Long, Long> statisticsMerged, final Map<Long, Double> sizeMerged) throws IOException {
+        if (segmentList == null || segmentList.isEmpty()) {
+            return;
+        }
+        int nSegment = segmentList.size();
+
+        Map<Long, HLLCounter> cuboidHLLMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
+        Map<Long, Double> sizeMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
+        for (CubeSegment pSegment : segmentList) {
+            CubeStatsReader pReader = new CubeStatsReader(pSegment, pSegment.getConfig());
+            Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowEstimatesHLLOrigin();
+            if (pHLLMap == null || pHLLMap.isEmpty()) {
+                logger.info("Cuboid Statistics for segment " + pSegment.getName() + " is not enabled.");
+                nSegment--;
+                continue;
+            }
+            Map<Long, Double> pSizeMap = pReader.getCuboidSizeMap();
+            for (Long pCuboid : cuboidSet) {
+                HLLCounter pInnerHLL = pHLLMap.get(pCuboid);
+                Preconditions.checkNotNull(pInnerHLL, "statistics should exist for cuboid " + pCuboid + " of segment "
+                        + pSegment.getCubeDesc().getName() + "[" + pSegment.getName() + "]");
+                if (cuboidHLLMapMerged.get(pCuboid) != null) {
+                    cuboidHLLMapMerged.get(pCuboid).merge(pInnerHLL);
+                } else {
+                    cuboidHLLMapMerged.put(pCuboid, pInnerHLL);
+                }
+
+                Double pSize = sizeMapMerged.get(pCuboid);
+                sizeMapMerged.put(pCuboid, pSize == null ? pSizeMap.get(pCuboid) : pSizeMap.get(pCuboid) + pSize);
+            }
+        }
+
+        if (nSegment < 1) {
+            return;
+        }
+        for (Long pCuboid : cuboidSet) {
+            statisticsMerged.put(pCuboid, cuboidHLLMapMerged.get(pCuboid).getCountEstimate() / nSegment);
+            sizeMerged.put(pCuboid, sizeMapMerged.get(pCuboid) / nSegment);
+        }
+    }
+
+    public static Map<Long, Long> readCuboidStatsFromSegment(Set<Long> cuboidIds, CubeSegment cubeSegment)
+            throws IOException {
+        if (cubeSegment == null) {
+            logger.warn("The cube segment can not be " + null);
+            return null;
+        }
+
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, cubeSegment.getConfig());
+        if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+                || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+            logger.info("Cuboid Statistics is not enabled.");
+            return null;
+        }
+
+        Map<Long, Long> cuboidsWithStatsAll = cubeStatsReader.getCuboidRowEstimatesHLL();
+        Map<Long, Long> cuboidsWithStats = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        for (Long cuboid : cuboidIds) {
+            Long rowEstimate = cuboidsWithStatsAll.get(cuboid);
+            if (rowEstimate == null) {
+                logger.warn("Cannot get the row count stats for cuboid " + cuboid);
+            } else {
+                cuboidsWithStats.put(cuboid, rowEstimate);
+            }
+        }
+        return cuboidsWithStats;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
new file mode 100644
index 0000000..cfac0e7
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StatisticsDecisionUtil {
+    protected static final Logger logger = LoggerFactory.getLogger(StatisticsDecisionUtil.class);
+
+    public static void decideCubingAlgorithm(CubingJob cubingJob, CubeSegment seg) throws IOException {
+        CubeStatsReader cubeStats = new CubeStatsReader(seg, seg.getConfig());
+        decideCubingAlgorithm(cubingJob, seg, cubeStats.getMapperOverlapRatioOfFirstBuild(),
+                cubeStats.getMapperNumberOfFirstBuild());
+    }
+
+    public static void decideCubingAlgorithm(CubingJob cubingJob, CubeSegment seg, double mapperOverlapRatio,
+            int mapperNumber) throws IOException {
+        KylinConfig kylinConf = seg.getConfig();
+        String algPref = kylinConf.getCubeAlgorithm();
+        CubingJob.AlgorithmEnum alg;
+        if (CubingJob.AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
+            alg = CubingJob.AlgorithmEnum.INMEM;
+        } else if (CubingJob.AlgorithmEnum.LAYER.name().equalsIgnoreCase(algPref)) {
+            alg = CubingJob.AlgorithmEnum.LAYER;
+        } else {
+            int memoryHungryMeasures = 0;
+            for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) {
+                if (measure.getFunction().getMeasureType().isMemoryHungry()) {
+                    logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression());
+                    memoryHungryMeasures++;
+                }
+            }
+
+            if (memoryHungryMeasures > 0) {
+                alg = CubingJob.AlgorithmEnum.LAYER;
+            } else if ("random".equalsIgnoreCase(algPref)) { // for testing
+                alg = new Random().nextBoolean() ? CubingJob.AlgorithmEnum.INMEM : CubingJob.AlgorithmEnum.LAYER;
+            } else { // the default
+                int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
+                double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
+                logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
+                logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is "
+                        + overlapThreshold);
+
+                // in-mem cubing is good when
+                // 1) the cluster has enough mapper slots to run in parallel
+                // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
+                alg = (mapperNumber <= mapperNumLimit && mapperOverlapRatio <= overlapThreshold)//
+                        ? CubingJob.AlgorithmEnum.INMEM
+                        : CubingJob.AlgorithmEnum.LAYER;
+            }
+
+        }
+        logger.info("The cube algorithm for " + seg + " is " + alg);
+
+        cubingJob.setAlgorithm(alg);
+    }
+
+    public static void optimizeCubingPlan(CubeSegment segment) throws IOException {
+        CubeInstance cube = segment.getCubeInstance();
+        List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY);
+        if (readySegments.size() == 0
+                || (cube.getConfig().isCubePlannerEnabledForExistingCube() && readySegments.size() == 1
+                        && (readySegments.get(0).getDateRangeStart() == segment.getDateRangeStart())
+                        && (readySegments.get(0).getDateRangeEnd() == segment.getDateRangeEnd()))) {
+            logger.info("It's able to trigger cuboid planner algorithm.");
+        } else {
+            return;
+        }
+
+        Map<Long, Long> recommendCuboidsWithStats = CuboidRecommenderUtil.getRecommendCuboidList(segment);
+        if (recommendCuboidsWithStats == null || recommendCuboidsWithStats.isEmpty()) {
+            return;
+        }
+
+        CubeUpdate cubeBuilder = new CubeUpdate(cube);
+        cubeBuilder.setCuboids(recommendCuboidsWithStats);
+        CubeManager.getInstance(cube.getConfig()).updateCube(cubeBuilder);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed0cc11f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 7544188..5971b6c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,7 +19,6 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
-import java.util.Random;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -30,14 +29,12 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.HadoopUtil;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.engine.mr.common.StatisticsDecisionUtil;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
 import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.model.MeasureDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -71,12 +68,15 @@ public class SaveStatisticsStep extends AbstractExecutable {
                 // put the statistics to metadata store
                 String statisticsFileName = newSegment.getStatisticsResourcePath();
                 rs.putResource(statisticsFileName, is, System.currentTimeMillis());
+
+                CubingJob cubingJob = (CubingJob) getManager()
+                        .getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
+                StatisticsDecisionUtil.decideCubingAlgorithm(cubingJob, newSegment);
+                StatisticsDecisionUtil.optimizeCubingPlan(newSegment);
             } finally {
                 IOUtils.closeStream(is);
             }
 
-            decideCubingAlgorithm(newSegment, kylinConf);
-
             return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
         } catch (IOException e) {
             logger.error("fail to save cuboid statistics", e);
@@ -84,47 +84,4 @@ public class SaveStatisticsStep extends AbstractExecutable {
         }
     }
 
-    private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) throws IOException {
-        String algPref = kylinConf.getCubeAlgorithm();
-        AlgorithmEnum alg;
-        if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
-            alg = AlgorithmEnum.INMEM;
-        } else if (AlgorithmEnum.LAYER.name().equalsIgnoreCase(algPref)) {
-            alg = AlgorithmEnum.LAYER;
-        } else {
-            int memoryHungryMeasures = 0;
-            for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) {
-                if (measure.getFunction().getMeasureType().isMemoryHungry()) {
-                    logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression());
-                    memoryHungryMeasures++;
-                }
-            }
-
-            if (memoryHungryMeasures > 0) {
-                alg = AlgorithmEnum.LAYER;
-            } else if ("random".equalsIgnoreCase(algPref)) { // for testing
-                alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
-            } else { // the default
-                CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
-                int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
-                int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
-                double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
-                double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
-                logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
-                logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
-                // in-mem cubing is good when
-                // 1) the cluster has enough mapper slots to run in parallel
-                // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
-                alg = (mapperNumber <= mapperNumLimit && mapperOverlapRatio <= overlapThreshold)//
-                        ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
-            }
-
-        }
-        logger.info("The cube algorithm for " + seg + " is " + alg);
-
-        CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
-        cubingJob.setAlgorithm(alg);
-    }
-
 }


[09/22] kylin git commit: APACHE-KYLIN-2731: Introduce checkpoint executable

Posted by nj...@apache.org.
APACHE-KYLIN-2731: Introduce checkpoint executable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05ce0358
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05ce0358
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05ce0358

Branch: refs/heads/yaho-cube-planner
Commit: 05ce03589369fb2946e0ff5b4eb072e143d401c3
Parents: 22166fd
Author: Zhong <nj...@apache.org>
Authored: Fri Aug 25 11:35:26 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:39 2017 +0800

----------------------------------------------------------------------
 .../kylin/cube/model/CubeBuildTypeEnum.java     |   7 +-
 .../kylin/job/execution/AbstractExecutable.java |   5 +
 .../job/execution/CheckpointExecutable.java     |  78 +++++++++
 .../engine/mr/common/JobInfoConverter.java      |  41 ++++-
 .../apache/kylin/rest/service/JobService.java   | 166 ++++++++++++++++++-
 5 files changed, 285 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/05ce0358/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
index e3ae214..6a14025 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
@@ -35,5 +35,10 @@ public enum CubeBuildTypeEnum {
     /**
      * refresh segments
      */
-    REFRESH
+    REFRESH,
+
+    /**
+     * checkpoint for set of other jobs
+     */
+    CHECKPOINT
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/05ce0358/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index d36f598..30b6421 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -384,6 +384,11 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
         return getDuration(getStartTime(), getEndTime(), getInterruptTime());
     }
 
+    public boolean isReady() {
+        final Output output = getManager().getOutput(id);
+        return output.getState() == ExecutableState.READY;
+    }
+
     /*
     * discarded is triggered by JobService, the Scheduler is not awake of that
     *

http://git-wip-us.apache.org/repos/asf/kylin/blob/05ce0358/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
new file mode 100644
index 0000000..9864400
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.execution;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class CheckpointExecutable extends DefaultChainedExecutable {
+
+    private static final Logger logger = LoggerFactory.getLogger(CheckpointExecutable.class);
+
+    private static final String DEPLOY_ENV_NAME = "envName";
+    private static final String PROJECT_INSTANCE_NAME = "projectName";
+
+    private final List<AbstractExecutable> subTasksForCheck = Lists.newArrayList();
+
+    public void addTaskForCheck(AbstractExecutable executable) {
+        this.subTasksForCheck.add(executable);
+    }
+
+    public void addTaskListForCheck(List<AbstractExecutable> executableList) {
+        this.subTasksForCheck.addAll(executableList);
+    }
+
+    public List<AbstractExecutable> getSubTasksForCheck() {
+        return subTasksForCheck;
+    }
+
+    @Override
+    public boolean isReady() {
+        if (!super.isReady()) {
+            return false;
+        }
+        for (Executable task : subTasksForCheck) {
+            final Output output = getManager().getOutput(task.getId());
+            if (output.getState() != ExecutableState.SUCCEED) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public String getDeployEnvName() {
+        return getParam(DEPLOY_ENV_NAME);
+    }
+
+    public void setDeployEnvName(String name) {
+        setParam(DEPLOY_ENV_NAME, name);
+    }
+
+    public String getProjectName() {
+        return getParam(PROJECT_INSTANCE_NAME);
+    }
+
+    public void setProjectName(String name) {
+        setParam(PROJECT_INSTANCE_NAME, name);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/05ce0358/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
index c465e3f..20c5d39 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -28,26 +28,25 @@ import org.apache.kylin.job.common.ShellExecutable;
 import org.apache.kylin.job.constant.JobStatusEnum;
 import org.apache.kylin.job.constant.JobStepStatusEnum;
 import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.CheckpointExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.execution.Output;
 
 import com.google.common.base.Preconditions;
 
 public class JobInfoConverter {
-    public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
+    public static JobInstance parseToJobInstance(CubingJob job, Map<String, Output> outputs) {
         if (job == null) {
             return null;
         }
-        Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
-        CubingJob cubeJob = (CubingJob) job;
         Output output = outputs.get(job.getId());
         final JobInstance result = new JobInstance();
         result.setName(job.getName());
-        result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
-        result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
+        result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
+        result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams()));
         result.setLastModified(output.getLastModified());
-        result.setSubmitter(cubeJob.getSubmitter());
-        result.setUuid(cubeJob.getId());
+        result.setSubmitter(job.getSubmitter());
+        result.setUuid(job.getId());
         result.setType(CubeBuildTypeEnum.BUILD);
         result.setStatus(parseToJobStatus(output.getState()));
         result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
@@ -55,8 +54,32 @@ public class JobInfoConverter {
         result.setExecEndTime(AbstractExecutable.getEndTime(output));
         result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
         result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(), result.getExecInterruptTime()) / 1000);
-        for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
-            AbstractExecutable task = cubeJob.getTasks().get(i);
+        for (int i = 0; i < job.getTasks().size(); ++i) {
+            AbstractExecutable task = job.getTasks().get(i);
+            result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
+        }
+        return result;
+    }
+
+    public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) {
+        if (job == null) {
+            return null;
+        }
+        Output output = outputs.get(job.getId());
+        final JobInstance result = new JobInstance();
+        result.setName(job.getName());
+        result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
+        result.setLastModified(output.getLastModified());
+        result.setSubmitter(job.getSubmitter());
+        result.setUuid(job.getId());
+        result.setType(CubeBuildTypeEnum.CHECKPOINT);
+        result.setStatus(parseToJobStatus(output.getState()));
+        result.setExecStartTime(AbstractExecutable.getStartTime(output));
+        result.setExecEndTime(AbstractExecutable.getEndTime(output));
+        result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
+        result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(), result.getExecInterruptTime()) / 1000);
+        for (int i = 0; i < job.getTasks().size(); ++i) {
+            AbstractExecutable task = job.getTasks().get(i);
             result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
         }
         return result;

http://git-wip-us.apache.org/repos/asf/kylin/blob/05ce0358/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 5bf684a..432d300 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -50,6 +50,7 @@ import org.apache.kylin.job.constant.JobTimeFilterEnum;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.job.exception.SchedulerException;
 import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.CheckpointExecutable;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.execution.Output;
@@ -315,6 +316,33 @@ public class JobService extends BasicService implements InitializingBean {
         return result;
     }
 
+    protected JobInstance getCheckpointJobInstance(AbstractExecutable job) {
+        Message msg = MsgPicker.getMsg();
+
+        if (job == null) {
+            return null;
+        }
+        if (!(job instanceof CheckpointExecutable)) {
+            throw new BadRequestException(String.format(msg.getILLEGAL_JOB_TYPE(), job.getId()));
+        }
+
+        CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job;
+        final JobInstance result = new JobInstance();
+        result.setName(job.getName());
+        result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
+        result.setLastModified(job.getLastModified());
+        result.setSubmitter(job.getSubmitter());
+        result.setUuid(job.getId());
+        result.setType(CubeBuildTypeEnum.CHECKPOINT);
+        result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
+        result.setDuration(job.getDuration() / 1000);
+        for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) {
+            AbstractExecutable task = checkpointExecutable.getTasks().get(i);
+            result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
+        }
+        return result;
+    }
+
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
             + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
     public void resumeJob(JobInstance job) {
@@ -374,6 +402,7 @@ public class JobService extends BasicService implements InitializingBean {
         Integer limit = (null == limitValue) ? 30 : limitValue;
         Integer offset = (null == offsetValue) ? 0 : offsetValue;
         List<JobInstance> jobs = searchJobsByCubeName(cubeNameSubstring, projectName, statusList, timeFilter);
+
         Collections.sort(jobs);
 
         if (jobs.size() <= offset) {
@@ -389,12 +418,40 @@ public class JobService extends BasicService implements InitializingBean {
 
     public List<JobInstance> searchJobsByCubeName(final String cubeNameSubstring, final String projectName,
             final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
-        return innerSearchCubingJobs(cubeNameSubstring, null, projectName, statusList, timeFilter);
+        return searchJobsByCubeName(cubeNameSubstring, projectName, statusList, timeFilter, JobSearchMode.ALL);
+    }
+
+    public List<JobInstance> searchJobsByCubeName(final String cubeNameSubstring, final String projectName,
+            final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter, JobSearchMode jobSearchMode) {
+        return innerSearchJobs(cubeNameSubstring, null, projectName, statusList, timeFilter, jobSearchMode);
     }
 
     public List<JobInstance> searchJobsByJobName(final String jobName, final String projectName,
             final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
-        return innerSearchCubingJobs(null, jobName, projectName, statusList, timeFilter);
+        return searchJobsByJobName(jobName, projectName, statusList, timeFilter, JobSearchMode.ALL);
+    }
+
+    public List<JobInstance> searchJobsByJobName(final String jobName, final String projectName,
+            final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter, JobSearchMode jobSearchMode) {
+        return innerSearchJobs(null, jobName, projectName, statusList, timeFilter, jobSearchMode);
+    }
+
+    public List<JobInstance> innerSearchJobs(final String cubeName, final String jobName, final String projectName,
+            final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter, JobSearchMode jobSearchMode) {
+        List<JobInstance> result = Lists.newArrayList();
+        switch (jobSearchMode) {
+        case CUBING_ONLY:
+            result.addAll(innerSearchCubingJobs(cubeName, jobName, projectName, statusList, timeFilter));
+            break;
+        case CHECKPOINT_ONLY:
+            result.addAll(innerSearchCheckpointJobs(cubeName, jobName, projectName, statusList, timeFilter));
+            break;
+        case ALL:
+        default:
+            result.addAll(innerSearchCubingJobs(cubeName, jobName, projectName, statusList, timeFilter));
+            result.addAll(innerSearchCheckpointJobs(cubeName, jobName, projectName, statusList, timeFilter));
+        }
+        return result;
     }
 
     public List<JobInstance> innerSearchCubingJobs(final String cubeName, final String jobName,
@@ -494,6 +551,108 @@ public class JobService extends BasicService implements InitializingBean {
         return results;
     }
 
+    public List<JobInstance> innerSearchCheckpointJobs(final String cubeName, final String jobName,
+            final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
+        // prepare time range
+        Calendar calendar = Calendar.getInstance();
+        calendar.setTime(new Date());
+        long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
+        long timeEndInMillis = Long.MAX_VALUE;
+        Set<ExecutableState> states = convertStatusEnumToStates(statusList);
+        final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
+
+        return Lists
+                .newArrayList(FluentIterable
+                        .from(innerSearchCheckpointJobs(cubeName, jobName, states, timeStartInMillis, timeEndInMillis,
+                                allOutputs, false, projectName))
+                        .transform(new Function<CheckpointExecutable, JobInstance>() {
+                            @Override
+                            public JobInstance apply(CheckpointExecutable checkpointExecutable) {
+                                return JobInfoConverter.parseToJobInstance(checkpointExecutable, allOutputs);
+                            }
+                        }));
+    }
+
+    public List<CheckpointExecutable> innerSearchCheckpointJobs(final String cubeName, final String jobName,
+            final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis,
+            final Map<String, Output> allOutputs, final boolean nameExactMatch, final String projectName) {
+        List<CheckpointExecutable> results = Lists
+                .newArrayList(
+                        FluentIterable
+                                .from(getExecutableManager().getAllAbstractExecutables(timeStartInMillis,
+                                        timeEndInMillis, CheckpointExecutable.class))
+                                .filter(new Predicate<AbstractExecutable>() {
+                                    @Override
+                                    public boolean apply(AbstractExecutable executable) {
+                                        if (executable instanceof CheckpointExecutable) {
+                                            if (StringUtils.isEmpty(cubeName)) {
+                                                return true;
+                                            }
+                                            String executableCubeName = CubingExecutableUtil
+                                                    .getCubeName(executable.getParams());
+                                            if (executableCubeName == null)
+                                                return true;
+                                            if (nameExactMatch)
+                                                return executableCubeName.toLowerCase().equals(cubeName);
+                                            else
+                                                return executableCubeName.toLowerCase()
+                                                        .contains(cubeName.toLowerCase());
+                                        } else {
+                                            return false;
+                                        }
+                                    }
+                                }).transform(new Function<AbstractExecutable, CheckpointExecutable>() {
+                                    @Override
+                                    public CheckpointExecutable apply(AbstractExecutable executable) {
+                                        return (CheckpointExecutable) executable;
+                                    }
+                                }).filter(Predicates.and(new Predicate<CheckpointExecutable>() {
+                                    @Override
+                                    public boolean apply(CheckpointExecutable executable) {
+                                        if (null == projectName
+                                                || null == getProjectManager().getProject(projectName)) {
+                                            return true;
+                                        } else {
+                                            return projectName.equals(executable.getProjectName());
+                                        }
+                                    }
+                                }, new Predicate<CheckpointExecutable>() {
+                                    @Override
+                                    public boolean apply(CheckpointExecutable executable) {
+                                        try {
+                                            Output output = allOutputs.get(executable.getId());
+                                            if (output == null) {
+                                                return false;
+                                            }
+
+                                            ExecutableState state = output.getState();
+                                            boolean ret = statusList.contains(state);
+                                            return ret;
+                                        } catch (Exception e) {
+                                            throw e;
+                                        }
+                                    }
+                                }, new Predicate<CheckpointExecutable>() {
+                                    @Override
+                                    public boolean apply(@Nullable CheckpointExecutable checkpointExecutable) {
+                                        if (checkpointExecutable == null) {
+                                            return false;
+                                        }
+
+                                        if (Strings.isEmpty(jobName)) {
+                                            return true;
+                                        }
+
+                                        if (nameExactMatch) {
+                                            return checkpointExecutable.getName().toLowerCase().equals(jobName);
+                                        } else {
+                                            return checkpointExecutable.getName().toLowerCase().contains(jobName);
+                                        }
+                                    }
+                                })));
+        return results;
+    }
+
     public List<CubingJob> listJobsByRealizationName(final String realizationName, final String projectName,
             final Set<ExecutableState> statusList) {
         return innerSearchCubingJobs(realizationName, null, statusList, 0L, Long.MAX_VALUE,
@@ -504,4 +663,7 @@ public class JobService extends BasicService implements InitializingBean {
         return listJobsByRealizationName(realizationName, projectName, EnumSet.allOf(ExecutableState.class));
     }
 
+    public enum JobSearchMode {
+        CUBING_ONLY, CHECKPOINT_ONLY, ALL
+    }
 }


[22/22] kylin git commit: APACHE-KYLIN-2826: change hashmap to concurrentmap for the allDescendantsCache of CuboidStats

Posted by nj...@apache.org.
APACHE-KYLIN-2826: change hashmap to concurrentmap for the allDescendantsCache of CuboidStats


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/844a6fc7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/844a6fc7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/844a6fc7

Branch: refs/heads/yaho-cube-planner
Commit: 844a6fc766ce6a3516d55406fc240b26fc8fc1f0
Parents: bd78cf1
Author: Zhong <nj...@apache.org>
Authored: Mon Sep 4 20:40:27 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:05 2017 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/844a6fc7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
index e4b0167..a1c191e 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -192,7 +192,7 @@ public class CuboidStats {
         this.directChildrenCache = ImmutableMap.<Long, List<Long>> builder()
                 .putAll(CuboidStatsUtil.createDirectChildrenCache(statistics.keySet())).build();
 
-        this.allDescendantsCache = Maps.newHashMap();
+        this.allDescendantsCache = Maps.newConcurrentMap();
     }
 
     private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,


[11/22] kylin git commit: APACHE-KYLIN-2802: fix issue of CuboidStatsUtil.createAllDescendantsCache

Posted by nj...@apache.org.
APACHE-KYLIN-2802: fix issue of CuboidStatsUtil.createAllDescendantsCache


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22166fde
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22166fde
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22166fde

Branch: refs/heads/yaho-cube-planner
Commit: 22166fdefc2be6e66be7ee1f440c10ebee10740b
Parents: ed0cc11
Author: Zhong <nj...@apache.org>
Authored: Fri Aug 25 08:15:49 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:39 2017 +0800

----------------------------------------------------------------------
 .../cube/cuboid/algorithm/CuboidStats.java      |  27 +++-
 .../cube/cuboid/algorithm/CuboidStatsUtil.java  | 134 +++++++++++-------
 .../cuboid/algorithm/CuboidStatsUtilTest.java   |  45 ++++---
 .../engine/mr/common/CuboidRecommenderUtil.java |   2 +-
 .../engine/mr/common/CuboidStatsReaderUtil.java | 135 +++++++++++++++++++
 .../kylin/engine/mr/common/CuboidStatsUtil.java | 135 -------------------
 6 files changed, 273 insertions(+), 205 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/22166fde/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
index 1775d5a..e4b0167 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.cube.cuboid.algorithm;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -120,7 +121,8 @@ public class CuboidStats {
     private ImmutableMap<Long, Double> cuboidHitProbabilityMap;
     private ImmutableMap<Long, Long> cuboidScanCountMap;
 
-    private ImmutableMap<Long, Set<Long>> allDescendantsCache;
+    private ImmutableMap<Long, List<Long>> directChildrenCache;
+    private Map<Long, Set<Long>> allDescendantsCache;
 
     private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, Map<Long, Long> statistics,
             Map<Long, Double> size, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
@@ -187,8 +189,10 @@ public class CuboidStats {
         }
         this.cuboidScanCountMap = ImmutableMap.<Long, Long> builder().putAll(tmpCuboidScanCountMap).build();
 
-        this.allDescendantsCache = ImmutableMap.<Long, Set<Long>> builder()
-                .putAll(CuboidStatsUtil.createAllDescendantsCache(statistics.keySet())).build();
+        this.directChildrenCache = ImmutableMap.<Long, List<Long>> builder()
+                .putAll(CuboidStatsUtil.createDirectChildrenCache(statistics.keySet())).build();
+
+        this.allDescendantsCache = Maps.newHashMap();
     }
 
     private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
@@ -216,11 +220,26 @@ public class CuboidStats {
     public Set<Long> getAllDescendants(long cuboid) {
         Set<Long> allDescendants = Sets.newLinkedHashSet();
         if (selectionCuboidSet.contains(cuboid)) {
-            return allDescendantsCache.get(cuboid);
+            if (allDescendantsCache.get(cuboid) != null) {
+                return allDescendantsCache.get(cuboid);
+            } else {
+                getAllDescendants(cuboid, allDescendants);
+                allDescendantsCache.put(cuboid, allDescendants);
+            }
         }
         return allDescendants;
     }
 
+    private void getAllDescendants(long cuboid, Set<Long> allDescendants) {
+        if (allDescendants.contains(cuboid)) {
+            return;
+        }
+        allDescendants.add(cuboid);
+        for (Long directChild : directChildrenCache.get(cuboid)) {
+            getAllDescendants(directChild, allDescendants);
+        }
+    }
+
     public Set<Long> getAllCuboidsForSelection() {
         return selectionCuboidSet;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/22166fde/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
index 6d5bbe5..c2683df 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -20,13 +20,13 @@ package org.apache.kylin.cube.cuboid.algorithm;
 
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -102,62 +102,102 @@ public class CuboidStatsUtil {
         }
     }
 
-    /** Using dynamic programming to use extra space to reduce repetitive computation*/
-    public static Map<Long, Set<Long>> createAllDescendantsCache(final Set<Long> cuboidSet) {
-        List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
-        Collections.sort(latticeCuboidList);
-
-        Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
-        Set<Long> preNoneDescendants = Sets.newHashSet();
-        for (int i = 0; i < latticeCuboidList.size(); i++) {
-            Long currentCuboid = latticeCuboidList.get(i);
-            Set<Long> descendants = Sets.newHashSet(currentCuboid);
-            Set<Long> curNoneDescendants = Sets.newHashSet();
-            if (i > 0) {
-                long preCuboid = latticeCuboidList.get(i - 1);
-                if (isDescendant(preCuboid, currentCuboid)) {
-                    descendants.addAll(allDescendantsCache.get(preCuboid));
-                } else {
-                    curNoneDescendants.add(preCuboid);
-                    for (long cuboidToCheck : allDescendantsCache.get(preCuboid)) {
-                        if (isDescendant(cuboidToCheck, currentCuboid)) {
-                            descendants.addAll(allDescendantsCache.get(cuboidToCheck));
-                        }
-                    }
-                }
-            }
-            for (long cuboidToCheck : preNoneDescendants) {
-                if (isDescendant(cuboidToCheck, currentCuboid)) {
-                    descendants.addAll(allDescendantsCache.get(cuboidToCheck));
-                } else {
-                    curNoneDescendants.add(cuboidToCheck);
+    public static Map<Long, List<Long>> createDirectChildrenCache(final Set<Long> cuboidSet) {
+        /**
+         * Sort the list by ascending order:
+         * */
+        final List<Long> cuboidList = Lists.newArrayList(cuboidSet);
+        Collections.sort(cuboidList);
+        /**
+         * Sort the list by ascending order:
+         * 1. the more bit count of its value, the bigger
+         * 2. the larger of its value, the bigger
+         * */
+        List<Integer> layerIdxList = Lists.newArrayListWithExpectedSize(cuboidList.size());
+        for (int i = 0; i < cuboidList.size(); i++) {
+            layerIdxList.add(i);
+        }
+        Collections.sort(layerIdxList, new Comparator<Integer>() {
+            @Override
+            public int compare(Integer i1, Integer i2) {
+                Long o1 = cuboidList.get(i1);
+                Long o2 = cuboidList.get(i2);
+                int nBitDiff = Long.bitCount(o1) - Long.bitCount(o2);
+                if (nBitDiff != 0) {
+                    return nBitDiff;
                 }
+                return Long.compare(o1, o2);
             }
+        });
+        /**
+         * Construct an index array for pointing the position in layerIdxList
+         * (layerCuboidList is for speeding up continuous iteration)
+         * */
+        int[] toLayerIdxArray = new int[layerIdxList.size()];
+        final List<Long> layerCuboidList = Lists.newArrayListWithExpectedSize(cuboidList.size());
+        for (int i = 0; i < layerIdxList.size(); i++) {
+            int cuboidIdx = layerIdxList.get(i);
+            toLayerIdxArray[cuboidIdx] = i;
+            layerCuboidList.add(cuboidList.get(cuboidIdx));
+        }
 
-            allDescendantsCache.put(currentCuboid, descendants);
-            preNoneDescendants = curNoneDescendants;
+        int[] previousLayerLastIdxArray = new int[layerIdxList.size()];
+        int currentBitCount = 0;
+        int previousLayerLastIdx = -1;
+        for (int i = 0; i < layerIdxList.size(); i++) {
+            int cuboidIdx = layerIdxList.get(i);
+            int nBits = Long.bitCount(cuboidList.get(cuboidIdx));
+            if (nBits > currentBitCount) {
+                currentBitCount = nBits;
+                previousLayerLastIdx = i - 1;
+            }
+            previousLayerLastIdxArray[i] = previousLayerLastIdx;
         }
 
-        return allDescendantsCache;
+        Map<Long, List<Long>> directChildrenCache = Maps.newHashMap();
+        for (int i = 0; i < cuboidList.size(); i++) {
+            Long currentCuboid = cuboidList.get(i);
+            LinkedList<Long> directChildren = Lists.newLinkedList();
+            int lastLayerIdx = previousLayerLastIdxArray[toLayerIdxArray[i]];
+            /**
+             * Choose one of the two scan strategies
+             * 1. cuboids are sorted by its value, like 1,2,3,4,...
+             * 2. cuboids are layered and sorted, like 1,2,4,8,...,3,5,...
+             * */
+            if (i - 1 <= lastLayerIdx) {
+                /**
+                 * 1. Adding cuboid by descending order
+                 * */
+                for (int j = i - 1; j >= 0; j--) {
+                    checkAndAddDirectChild(directChildren, currentCuboid, cuboidList.get(j));
+                }
+            } else {
+                /**
+                 * 1. Adding cuboid by descending order
+                 * 2. Check from lower cuboid layer
+                 * */
+                for (int j = lastLayerIdx; j >= 0; j--) {
+                    checkAndAddDirectChild(directChildren, currentCuboid, layerCuboidList.get(j));
+                }
+            }
+            directChildrenCache.put(currentCuboid, directChildren);
+        }
+        return directChildrenCache;
     }
 
-    @VisibleForTesting
-    static Map<Long, Set<Long>> createAllDescendantsCache2(final Set<Long> cuboidSet) {
-        List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
-
-        Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
-        for (int i = 0; i < latticeCuboidList.size(); i++) {
-            Long currentCuboid = latticeCuboidList.get(i);
-            Set<Long> descendantSet = Sets.newHashSet(currentCuboid);
-            for (int j = 0; j < i; j++) {
-                Long checkCuboid = latticeCuboidList.get(j);
-                if (isDescendant(checkCuboid, currentCuboid)) {
-                    descendantSet.add(checkCuboid);
+    private static void checkAndAddDirectChild(List<Long> directChildren, Long currentCuboid, Long checkedCuboid) {
+        if (isDescendant(checkedCuboid, currentCuboid)) {
+            boolean ifDirectChild = true;
+            for (long directChild : directChildren) {
+                if (isDescendant(checkedCuboid, directChild)) {
+                    ifDirectChild = false;
+                    break;
                 }
             }
-            allDescendantsCache.put(currentCuboid, descendantSet);
+            if (ifDirectChild) {
+                directChildren.add(checkedCuboid);
+            }
         }
-        return allDescendantsCache;
     }
 
     public static boolean isDescendant(long cuboidToCheck, long parentCuboid) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/22166fde/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index 8e809cf..688b539 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -19,14 +19,15 @@
 package org.apache.kylin.cube.cuboid.algorithm;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
 import org.junit.Assert;
 import org.junit.Test;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
@@ -122,16 +123,16 @@ public class CuboidStatsUtilTest {
     }
 
     @Test
-    public void createAllDescendantsCacheTest() {
+    public void createDirectChildrenCacheTest() {
         Set<Long> cuboidSet = generateCuboidSet();
-        Map<Long, Set<Long>> allDescendantsCache = CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
-
-        Assert.assertTrue(allDescendantsCache.get(255L).containsAll(cuboidSet));
-
-        Assert.assertTrue(allDescendantsCache.get(239L).size() == 5);
-
-        Assert.assertTrue(allDescendantsCache.get(50L).containsAll(Sets.newHashSet(50L, 2L)));
-        Assert.assertTrue(!allDescendantsCache.get(50L).contains(4L));
+        Map<Long, List<Long>> directChildrenCache = CuboidStatsUtil.createDirectChildrenCache(cuboidSet);
+
+        Assert.assertTrue(directChildrenCache.get(255L).containsAll(Lists.newArrayList(239L, 159L, 50L)));
+        Assert.assertTrue(directChildrenCache.get(159L).contains(6L));
+        Assert.assertTrue(directChildrenCache.get(50L).contains(2L));
+        Assert.assertTrue(directChildrenCache.get(239L).contains(199L));
+        Assert.assertTrue(directChildrenCache.get(199L).contains(6L));
+        Assert.assertTrue(directChildrenCache.get(6L).containsAll(Lists.newArrayList(4L, 2L)));
     }
 
     private Set<Long> generateMassCuboidSet() {
@@ -144,18 +145,26 @@ public class CuboidStatsUtilTest {
     }
 
     @Test
-    public void createAllDescendantsCacheStressTest() {
+    public void createDirectChildrenCacheStressTest() {
         Stopwatch sw = new Stopwatch();
         sw.start();
         Set<Long> cuboidSet = generateMassCuboidSet();
-        System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS));
-        sw.reset();
-        sw.start();
-        CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
-        System.out.println("Time elapsed for creating descendants cache: " + sw.elapsed(TimeUnit.MILLISECONDS));
+        System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsedMillis());
         sw.reset();
         sw.start();
-        CuboidStatsUtil.createAllDescendantsCache2(cuboidSet);
-        System.out.println("Time elapsed for creating descendants cache2: " + sw.elapsed(TimeUnit.MILLISECONDS));
+        checkDirectChildrenCacheStressTest(CuboidStatsUtil.createDirectChildrenCache(cuboidSet));
+        System.out.println("Time elapsed for creating direct children cache: " + sw.elapsedMillis());
+        sw.stop();
+    }
+
+    private void checkDirectChildrenCacheStressTest(Map<Long, List<Long>> directChildrenCache) {
+        for (Map.Entry<Long, List<Long>> entry : directChildrenCache.entrySet()) {
+            if (Long.bitCount(entry.getKey()) == 1) {
+                Assert.assertTrue("Check for cuboid " + entry.getKey(), entry.getValue().size() == 0);
+            } else {
+                Assert.assertTrue("Check for cuboid " + entry.getKey(),
+                        Long.bitCount(entry.getKey()) == entry.getValue().size());
+            }
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/22166fde/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
index a0b20fa..ba3f023 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -62,7 +62,7 @@ public class CuboidRecommenderUtil {
     public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
             Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
 
-        Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsUtil
+        Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsReaderUtil
                 .readCuboidStatsAndSizeFromCube(cube.getCuboidScheduler().getAllCuboidIds(), cube);
 
         String key = cube.getName();

http://git-wip-us.apache.org/repos/asf/kylin/blob/22166fde/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
new file mode 100644
index 0000000..aaf9aa3
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class CuboidStatsReaderUtil {
+
+    private static final Logger logger = LoggerFactory.getLogger(CuboidStatsReaderUtil.class);
+
+    public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance)
+            throws IOException {
+        Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+        return statisticsMerged.isEmpty() ? null : statisticsMerged;
+    }
+
+    public static Pair<Map<Long, Long>, Map<Long, Double>> readCuboidStatsAndSizeFromCube(Set<Long> cuboidIds,
+            CubeInstance cube) throws IOException {
+        Preconditions.checkNotNull(cuboidIds, "The cuboid set can not be null");
+        Preconditions.checkNotNull(cube, "The cube instance can not be null");
+
+        List<CubeSegment> segmentList = cube.getSegments(SegmentStatusEnum.READY);
+        Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        Map<Long, Double> sizeMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged, sizeMerged);
+        return new Pair<>(statisticsMerged, sizeMerged);
+    }
+
+    public static Map<Long, Long> readCuboidStatsFromSegments(Set<Long> cuboidIds, List<CubeSegment> segmentList)
+            throws IOException {
+        Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged,
+                Maps.<Long, Double> newHashMapWithExpectedSize(cuboidIds.size()));
+        return statisticsMerged.isEmpty() ? null : statisticsMerged;
+    }
+
+    private static void readCuboidStatsFromSegments(Set<Long> cuboidSet, List<CubeSegment> segmentList,
+            final Map<Long, Long> statisticsMerged, final Map<Long, Double> sizeMerged) throws IOException {
+        if (segmentList == null || segmentList.isEmpty()) {
+            return;
+        }
+        int nSegment = segmentList.size();
+
+        Map<Long, HLLCounter> cuboidHLLMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
+        Map<Long, Double> sizeMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
+        for (CubeSegment pSegment : segmentList) {
+            CubeStatsReader pReader = new CubeStatsReader(pSegment, pSegment.getConfig());
+            Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowEstimatesHLLOrigin();
+            if (pHLLMap == null || pHLLMap.isEmpty()) {
+                logger.info("Cuboid Statistics for segment " + pSegment.getName() + " is not enabled.");
+                nSegment--;
+                continue;
+            }
+            Map<Long, Double> pSizeMap = pReader.getCuboidSizeMap();
+            for (Long pCuboid : cuboidSet) {
+                HLLCounter pInnerHLL = pHLLMap.get(pCuboid);
+                Preconditions.checkNotNull(pInnerHLL, "statistics should exist for cuboid " + pCuboid + " of segment "
+                        + pSegment.getCubeDesc().getName() + "[" + pSegment.getName() + "]");
+                if (cuboidHLLMapMerged.get(pCuboid) != null) {
+                    cuboidHLLMapMerged.get(pCuboid).merge(pInnerHLL);
+                } else {
+                    cuboidHLLMapMerged.put(pCuboid, pInnerHLL);
+                }
+
+                Double pSize = sizeMapMerged.get(pCuboid);
+                sizeMapMerged.put(pCuboid, pSize == null ? pSizeMap.get(pCuboid) : pSizeMap.get(pCuboid) + pSize);
+            }
+        }
+
+        if (nSegment < 1) {
+            return;
+        }
+        for (Long pCuboid : cuboidSet) {
+            statisticsMerged.put(pCuboid, cuboidHLLMapMerged.get(pCuboid).getCountEstimate() / nSegment);
+            sizeMerged.put(pCuboid, sizeMapMerged.get(pCuboid) / nSegment);
+        }
+    }
+
+    public static Map<Long, Long> readCuboidStatsFromSegment(Set<Long> cuboidIds, CubeSegment cubeSegment)
+            throws IOException {
+        if (cubeSegment == null) {
+            logger.warn("The cube segment can not be " + null);
+            return null;
+        }
+
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, cubeSegment.getConfig());
+        if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+                || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+            logger.info("Cuboid Statistics is not enabled.");
+            return null;
+        }
+
+        Map<Long, Long> cuboidsWithStatsAll = cubeStatsReader.getCuboidRowEstimatesHLL();
+        Map<Long, Long> cuboidsWithStats = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+        for (Long cuboid : cuboidIds) {
+            Long rowEstimate = cuboidsWithStatsAll.get(cuboid);
+            if (rowEstimate == null) {
+                logger.warn("Cannot get the row count stats for cuboid " + cuboid);
+            } else {
+                cuboidsWithStats.put(cuboid, rowEstimate);
+            }
+        }
+        return cuboidsWithStats;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/22166fde/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
deleted file mode 100644
index 5bb4179..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.common;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class CuboidStatsUtil {
-
-    private static final Logger logger = LoggerFactory.getLogger(CuboidStatsUtil.class);
-
-    public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance)
-            throws IOException {
-        Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
-        return statisticsMerged.isEmpty() ? null : statisticsMerged;
-    }
-
-    public static Pair<Map<Long, Long>, Map<Long, Double>> readCuboidStatsAndSizeFromCube(Set<Long> cuboidIds,
-            CubeInstance cube) throws IOException {
-        Preconditions.checkNotNull(cuboidIds, "The cuboid set can not be null");
-        Preconditions.checkNotNull(cube, "The cube instance can not be null");
-
-        List<CubeSegment> segmentList = cube.getSegments(SegmentStatusEnum.READY);
-        Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
-        Map<Long, Double> sizeMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
-        readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged, sizeMerged);
-        return new Pair<>(statisticsMerged, sizeMerged);
-    }
-
-    public static Map<Long, Long> readCuboidStatsFromSegments(Set<Long> cuboidIds, List<CubeSegment> segmentList)
-            throws IOException {
-        Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
-        readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged,
-                Maps.<Long, Double> newHashMapWithExpectedSize(cuboidIds.size()));
-        return statisticsMerged.isEmpty() ? null : statisticsMerged;
-    }
-
-    private static void readCuboidStatsFromSegments(Set<Long> cuboidSet, List<CubeSegment> segmentList,
-            final Map<Long, Long> statisticsMerged, final Map<Long, Double> sizeMerged) throws IOException {
-        if (segmentList == null || segmentList.isEmpty()) {
-            return;
-        }
-        int nSegment = segmentList.size();
-
-        Map<Long, HLLCounter> cuboidHLLMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
-        Map<Long, Double> sizeMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
-        for (CubeSegment pSegment : segmentList) {
-            CubeStatsReader pReader = new CubeStatsReader(pSegment, pSegment.getConfig());
-            Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowEstimatesHLLOrigin();
-            if (pHLLMap == null || pHLLMap.isEmpty()) {
-                logger.info("Cuboid Statistics for segment " + pSegment.getName() + " is not enabled.");
-                nSegment--;
-                continue;
-            }
-            Map<Long, Double> pSizeMap = pReader.getCuboidSizeMap();
-            for (Long pCuboid : cuboidSet) {
-                HLLCounter pInnerHLL = pHLLMap.get(pCuboid);
-                Preconditions.checkNotNull(pInnerHLL, "statistics should exist for cuboid " + pCuboid + " of segment "
-                        + pSegment.getCubeDesc().getName() + "[" + pSegment.getName() + "]");
-                if (cuboidHLLMapMerged.get(pCuboid) != null) {
-                    cuboidHLLMapMerged.get(pCuboid).merge(pInnerHLL);
-                } else {
-                    cuboidHLLMapMerged.put(pCuboid, pInnerHLL);
-                }
-
-                Double pSize = sizeMapMerged.get(pCuboid);
-                sizeMapMerged.put(pCuboid, pSize == null ? pSizeMap.get(pCuboid) : pSizeMap.get(pCuboid) + pSize);
-            }
-        }
-
-        if (nSegment < 1) {
-            return;
-        }
-        for (Long pCuboid : cuboidSet) {
-            statisticsMerged.put(pCuboid, cuboidHLLMapMerged.get(pCuboid).getCountEstimate() / nSegment);
-            sizeMerged.put(pCuboid, sizeMapMerged.get(pCuboid) / nSegment);
-        }
-    }
-
-    public static Map<Long, Long> readCuboidStatsFromSegment(Set<Long> cuboidIds, CubeSegment cubeSegment)
-            throws IOException {
-        if (cubeSegment == null) {
-            logger.warn("The cube segment can not be " + null);
-            return null;
-        }
-
-        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, cubeSegment.getConfig());
-        if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
-                || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
-            logger.info("Cuboid Statistics is not enabled.");
-            return null;
-        }
-
-        Map<Long, Long> cuboidsWithStatsAll = cubeStatsReader.getCuboidRowEstimatesHLL();
-        Map<Long, Long> cuboidsWithStats = Maps.newHashMapWithExpectedSize(cuboidIds.size());
-        for (Long cuboid : cuboidIds) {
-            Long rowEstimate = cuboidsWithStatsAll.get(cuboid);
-            if (rowEstimate == null) {
-                logger.warn("Cannot get the row count stats for cuboid " + cuboid);
-            } else {
-                cuboidsWithStats.put(cuboid, rowEstimate);
-            }
-        }
-        return cuboidsWithStats;
-    }
-}


[19/22] kylin git commit: APACHE-KYLIN-2822: backend support for sunburst chart to show cuboid tree

Posted by nj...@apache.org.
APACHE-KYLIN-2822: backend support for sunburst chart to show cuboid tree


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab3ac1aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab3ac1aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab3ac1aa

Branch: refs/heads/yaho-cube-planner
Commit: ab3ac1aac33c6e2f4a0d4f21744b0287ebb7f571
Parents: 671cddd
Author: Zhong <nj...@apache.org>
Authored: Wed Aug 30 18:04:09 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:05 2017 +0800

----------------------------------------------------------------------
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  |   3 +-
 .../engine/mr/common/CuboidStatsReaderUtil.java |  10 +-
 .../kylin/rest/controller/CubeController.java   |  94 +++++++++++++-
 .../kylin/rest/response/CuboidTreeResponse.java | 123 +++++++++++++++++++
 .../apache/kylin/rest/service/CubeService.java  |  60 +++++++++
 5 files changed, 279 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab3ac1aa/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
index 0cf770d..8252e6c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -19,7 +19,6 @@
 package org.apache.kylin.cube.cuboid;
 
 import java.io.PrintWriter;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -243,7 +242,7 @@ public class TreeCuboidScheduler extends CuboidScheduler {
         @JsonIgnore
         int level;
         @JsonProperty("children")
-        List<TreeNode> children = new ArrayList<>();
+        List<TreeNode> children = Lists.newArrayList();
 
         public long getCuboidId() {
             return cuboidId;

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab3ac1aa/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
index bfb37ce..604f600 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -38,9 +38,13 @@ public class CuboidStatsReaderUtil {
 
     private static final Logger logger = LoggerFactory.getLogger(CuboidStatsReaderUtil.class);
 
-    public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance)
-            throws IOException {
-        Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+    public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance) {
+        Map<Long, Long> statisticsMerged = null;
+        try {
+            statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+        } catch (IOException e) {
+            logger.warn("Fail to read statistics for cube " + cubeInstance.getName() + " due to " + e);
+        }
         return statisticsMerged.isEmpty() ? null : statisticsMerged;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab3ac1aa/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index c415212..3118c59 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -38,11 +38,14 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.job.exception.JobException;
@@ -61,6 +64,7 @@ import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobBuildRequest2;
 import org.apache.kylin.rest.request.JobOptimizeRequest;
 import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.CuboidTreeResponse;
 import org.apache.kylin.rest.response.GeneralResponse;
 import org.apache.kylin.rest.response.HBaseResponse;
 import org.apache.kylin.rest.service.CubeService;
@@ -763,20 +767,81 @@ public class CubeController extends BasicController {
         }
     }
 
+    @RequestMapping(value = "/{cubeName}/cuboids/current", method = RequestMethod.GET)
+    @ResponseBody
+    public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName) {
+        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+        if (cube == null) {
+            logger.error("Get cube: [" + cubeName + "] failed when get current cuboids");
+            throw new BadRequestException("Get cube: [" + cubeName + "] failed when get current cuboids");
+        }
+        // The cuboid tree displayed should be consistent with the current one
+        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        Map<Long, Long> cuboidStatsMap = cube.getCuboids();
+        if (cuboidStatsMap == null) {
+            cuboidStatsMap = CuboidStatsReaderUtil.readCuboidStatsFromCube(cuboidScheduler.getAllCuboidIds(), cube);
+        }
+
+        Map<Long, Long> hitFrequencyMap = null;
+        Map<Long, Long> queryMatchMap = null;
+        try {
+            hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
+            queryMatchMap = getCuboidQueryMatchCount(cubeName);
+        } catch (Exception e) {
+            logger.warn("Fail to query on system cube due to " + e);
+        }
+
+        Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+        return cubeService.getCuboidTreeResponse(cuboidScheduler, cuboidStatsMap, hitFrequencyMap, queryMatchMap,
+                currentCuboidSet);
+    }
+
+    @RequestMapping(value = "/{cubeName}/cuboids/recommend", method = RequestMethod.GET)
+    @ResponseBody
+    public CuboidTreeResponse getRecommendCuboids(@PathVariable String cubeName) throws IOException {
+        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+        if (cube == null) {
+            logger.error("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+            throw new BadRequestException("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+        }
+        Map<Long, Long> recommendCuboidStatsMap = getRecommendCuboidList(cube);
+        if (recommendCuboidStatsMap == null || recommendCuboidStatsMap.isEmpty()) {
+            return new CuboidTreeResponse();
+        }
+        CuboidScheduler cuboidScheduler = new TreeCuboidScheduler(cube.getDescriptor(),
+                Lists.newArrayList(recommendCuboidStatsMap.keySet()),
+                new TreeCuboidScheduler.CuboidCostComparator(recommendCuboidStatsMap));
+
+        // Get cuboid target info for displaying heat map of cuboid hit
+        Map<Long, Long> displayHitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
+        // Get exactly matched cuboid query count
+        Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName);
+
+        Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+        return cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap, displayHitFrequencyMap,
+                queryMatchMap, currentCuboidSet);
+    }
+
     private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException {
         // Get cuboid source info
-        Map<Long, Long> optimizeHitFrequencyMap = getCuboidHitFrequency(cube.getName(), true);
+        Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName());
         Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName());
         return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap);
     }
 
-    private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
+    private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) {
+        return getCuboidHitFrequency(cubeName, true);
+    }
+
+    private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) {
+        return getCuboidHitFrequency(cubeName, false);
+    }
+
+    private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean ifSource) {
         SQLRequest sqlRequest = new SQLRequest();
         sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
-        String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
-        if (!isCuboidSource) {
-            cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
-        }
+        String cuboidColumn = ifSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString()
+                : QueryCubePropertyEnum.CUBOID_TARGET.toString();
         String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
         String table = cubeService.getMetricsManager()
                 .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
@@ -806,6 +871,23 @@ public class CubeController extends BasicController {
         return cubeService.formatRollingUpCount(orgRollingUpCount);
     }
 
+    private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
+        SQLRequest sqlRequest = new SQLRequest();
+        sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+        String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+        String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+        String table = cubeService.getMetricsManager()
+                .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
+        String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " //
+                + "from " + table //
+                + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' and "
+                + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " //
+                + "group by " + cuboidSource;
+        sqlRequest.setSql(sql);
+        List<List<String>> orgMatchHitFrequency = queryService.queryWithoutSecure(sqlRequest).getResults();
+        return cubeService.formatQueryCount(orgMatchHitFrequency);
+    }
+
     /**
      * Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
      * source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab3ac1aa/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java
new file mode 100644
index 0000000..b416084
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+import java.io.Serializable;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
+
+public class CuboidTreeResponse implements Serializable {
+
+    private static final long serialVersionUID = 2835980715891990832L;
+
+    private NodeInfo root;
+
+    public NodeInfo getRoot() {
+        return root;
+    }
+
+    public void setRoot(NodeInfo root) {
+        this.root = root;
+    }
+
+    public static class NodeInfo {
+        @JsonProperty("cuboid_id")
+        private Long id;
+        @JsonProperty("name")
+        private String name;
+        @JsonProperty("query_count")
+        private Long queryCount;
+        @JsonProperty("query_rate")
+        private Float queryRate;
+        @JsonProperty("exactly_match_count")
+        private Long exactlyMatchCount;
+        @JsonProperty("row_count")
+        private Long rowCount;
+        @JsonProperty("existed")
+        private Boolean existed;
+        @JsonProperty("children")
+        List<NodeInfo> children = Lists.newArrayList();
+
+        public Long getId() {
+            return id;
+        }
+
+        public void setId(Long id) {
+            this.id = id;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public Long getQueryCount() {
+            return queryCount;
+        }
+
+        public void setQueryCount(Long queryCount) {
+            this.queryCount = queryCount;
+        }
+
+        public Float getQueryRate() {
+            return queryRate;
+        }
+
+        public void setQueryRate(Float queryRate) {
+            this.queryRate = queryRate;
+        }
+
+        public Long getExactlyMatchCount() {
+            return exactlyMatchCount;
+        }
+
+        public void setExactlyMatchCount(Long exactlyMatchCount) {
+            this.exactlyMatchCount = exactlyMatchCount;
+        }
+
+        public Long getRowCount() {
+            return rowCount;
+        }
+
+        public void setRowCount(Long rowCount) {
+            this.rowCount = rowCount;
+        }
+
+        public Boolean getExisted() {
+            return existed;
+        }
+
+        public void setExisted(Boolean existed) {
+            this.existed = existed;
+        }
+
+        public void addChild(NodeInfo child) {
+            this.children.add(child);
+        }
+
+        public List<NodeInfo> getChildren() {
+            return children;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab3ac1aa/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index fdf7bec..7268209 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -25,6 +25,7 @@ import java.util.Date;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.WeakHashMap;
 
 import org.apache.commons.lang.StringUtils;
@@ -35,7 +36,9 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidCLI;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.engine.mr.CubingJob;
@@ -57,6 +60,8 @@ import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.msg.Message;
 import org.apache.kylin.rest.msg.MsgPicker;
 import org.apache.kylin.rest.request.MetricsRequest;
+import org.apache.kylin.rest.response.CuboidTreeResponse;
+import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
 import org.apache.kylin.rest.response.HBaseResponse;
 import org.apache.kylin.rest.response.MetricsResponse;
 import org.apache.kylin.rest.security.AclPermission;
@@ -768,6 +773,61 @@ public class CubeService extends BasicService {
         return formattedRollingUpCount;
     }
 
+    public CuboidTreeResponse getCuboidTreeResponse(CuboidScheduler cuboidScheduler, Map<Long, Long> rowCountMap,
+            Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap, Set<Long> currentCuboidSet) {
+        long baseCuboidId = cuboidScheduler.getBaseCuboidId();
+        int dimensionCount = Long.bitCount(baseCuboidId);
+
+        // get cube query count total
+        long cubeQueryCount = 0L;
+        if (hitFrequencyMap != null) {
+            for (long queryCount : hitFrequencyMap.values()) {
+                cubeQueryCount += queryCount;
+            }
+        }
+
+        NodeInfo root = generateNodeInfo(baseCuboidId, dimensionCount, cubeQueryCount, rowCountMap, hitFrequencyMap,
+                queryMatchMap, currentCuboidSet);
+
+        List<NodeInfo> nodeQueue = Lists.newLinkedList();
+        nodeQueue.add(root);
+        while (!nodeQueue.isEmpty()) {
+            NodeInfo parentNode = nodeQueue.remove(0);
+            for (long childId : cuboidScheduler.getSpanningCuboid(parentNode.getId())) {
+                NodeInfo childNode = generateNodeInfo(childId, dimensionCount, cubeQueryCount, rowCountMap,
+                        hitFrequencyMap, queryMatchMap, currentCuboidSet);
+                parentNode.addChild(childNode);
+                nodeQueue.add(childNode);
+            }
+        }
+
+        CuboidTreeResponse result = new CuboidTreeResponse();
+        result.setRoot(root);
+        return result;
+    }
+
+    private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQueryCount,
+            Map<Long, Long> rowCountMap, Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap,
+            Set<Long> currentCuboidSet) {
+        Long queryCount = hitFrequencyMap == null || hitFrequencyMap.get(cuboidId) == null ? 0L
+                : hitFrequencyMap.get(cuboidId);
+        float queryRate = cubeQueryCount <= 0 ? 0 : queryCount.floatValue() / cubeQueryCount;
+        long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L
+                : queryMatchMap.get(cuboidId);
+        boolean ifExist = currentCuboidSet.contains(cuboidId);
+        long rowCount = rowCountMap == null ? 0L : rowCountMap.get(cuboidId);
+
+        NodeInfo node = new NodeInfo();
+        node.setId(cuboidId);
+        node.setName(Cuboid.getDisplayName(cuboidId, dimensionCount));
+        node.setQueryCount(queryCount);
+        node.setQueryRate(queryRate);
+        node.setExactlyMatchCount(queryExactlyMatchCount);
+        node.setExisted(ifExist);
+        node.setRowCount(rowCount);
+        return node;
+    }
+
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')")
     public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
             Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {


[02/22] kylin git commit: APACHE-KYLIN-2729: Introduce greedy algorithm for cube planner

Posted by nj...@apache.org.
APACHE-KYLIN-2729: Introduce greedy algorithm for cube planner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74772d78
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74772d78
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74772d78

Branch: refs/heads/yaho-cube-planner
Commit: 74772d78ccbf1e000eee534f57eda2460ae653bf
Parents: 03e4586
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 14:58:18 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:38 2017 +0800

----------------------------------------------------------------------
 .../algorithm/greedy/GreedyAlgorithm.java       | 170 +++++++++++++++++++
 .../cuboid/algorithm/GreedyAlgorithmTest.java   |  58 +++++++
 2 files changed, 228 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/74772d78/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
new file mode 100755
index 0000000..8d788b1
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.greedy;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.kylin.cube.cuboid.algorithm.AbstractRecommendAlgorithm;
+import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidBenefitModel;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+* A simple implementation of the Greedy Algorithm , it chooses the cuboids which give
+* the greatest benefit based on expansion rate and time limitation.
+*/
+public class GreedyAlgorithm extends AbstractRecommendAlgorithm {
+    private static final Logger logger = LoggerFactory.getLogger(GreedyAlgorithm.class);
+
+    private static final int THREAD_NUM = 8;
+    private ExecutorService executor;
+
+    private Set<Long> selected = Sets.newLinkedHashSet();
+    private List<Long> remaining = Lists.newLinkedList();
+
+    public GreedyAlgorithm(final long timeout, BenefitPolicy benefitPolicy, CuboidStats cuboidStats) {
+        super(timeout, benefitPolicy, cuboidStats);
+    }
+
+    @Override
+    public List<Long> recommend(double expansionRate) {
+        double spaceLimit = getCuboidStats().getBaseCuboidSize() * expansionRate;
+        return start(spaceLimit);
+    }
+
+    @Override
+    public List<Long> start(double spaceLimit) {
+        logger.info("Greedy Algorithm started.");
+        executor = Executors.newFixedThreadPool(THREAD_NUM,
+                new ThreadFactoryBuilder().setNameFormat("greedy-algorithm-benefit-calculator-pool-%d").build());
+
+        getBenefitPolicy().initBeforeStart();
+
+        //Initial mandatory cuboids
+        selected.clear();
+        double remainingSpace = spaceLimit;
+        for (Long mandatoryOne : getCuboidStats().getAllCuboidsForMandatory()) {
+            selected.add(mandatoryOne);
+            if (getCuboidStats().getCuboidSize(mandatoryOne) != null) {
+                remainingSpace -= getCuboidStats().getCuboidSize(mandatoryOne);
+            }
+        }
+        //Initial remaining cuboid set
+        remaining.clear();
+        remaining.addAll(getCuboidStats().getAllCuboidsForSelection());
+
+        long round = 0;
+        while (true) {
+            if (shouldCancel()) {
+                break;
+            }
+            // Choose one cuboId having the maximum benefit per unit space in all available list
+            CuboidBenefitModel best = recommendBestOne();
+            // If return null, then we should finish the process and return
+            if (best == null) {
+                break;
+            }
+            // If we finally find the cuboid selected does not meet a minimum threshold of benefit (for
+            // example, a cuboid with 0.99M roll up from a parent cuboid with 1M
+            // rows), then we should finish the process and return
+            if (!getBenefitPolicy().ifEfficient(best)) {
+                break;
+            }
+
+            remainingSpace -= getCuboidStats().getCuboidSize(best.getCuboidId());
+            // If we finally find there is no remaining space,  then we should finish the process and return
+            if (remainingSpace <= 0) {
+                break;
+            }
+            selected.add(best.getCuboidId());
+            remaining.remove(best.getCuboidId());
+            getBenefitPolicy().propagateAggregationCost(best.getCuboidId(), selected);
+            round++;
+            if (logger.isInfoEnabled()) {
+                logger.info(String.format("Recommend in round %d : %s", round, best.toString()));
+            }
+        }
+
+        executor.shutdown();
+
+        List<Long> excluded = Lists.newArrayList(remaining);
+        remaining.retainAll(selected);
+        Preconditions.checkArgument(remaining.size() == 0,
+                "There should be no intersection between excluded list and selected list.");
+        logger.info("Greedy Algorithm finished.");
+
+        if (logger.isInfoEnabled()) {
+            logger.info("Excluded cuboidId size:" + excluded.size());
+            logger.info("Excluded cuboidId detail:");
+            for (Long cuboid : excluded) {
+                logger.info(String.format("cuboidId %d and Cost: %d and Space: %f", cuboid,
+                        getCuboidStats().getCuboidQueryCost(cuboid), getCuboidStats().getCuboidSize(cuboid)));
+            }
+            logger.info("Total Space:" + (spaceLimit - remainingSpace));
+            logger.info("Space Expansion Rate:" + (spaceLimit - remainingSpace) / getCuboidStats().getBaseCuboidSize());
+        }
+        return Lists.newArrayList(selected);
+    }
+
+    private CuboidBenefitModel recommendBestOne() {
+        final int selectedSize = selected.size();
+        final AtomicReference<CuboidBenefitModel> best = new AtomicReference<>();
+
+        final CountDownLatch counter = new CountDownLatch(remaining.size());
+        for (final Long cuboid : remaining) {
+            executor.submit(new Runnable() {
+                @Override
+                public void run() {
+                    CuboidBenefitModel currentBest = best.get();
+                    assert (selected.size() == selectedSize);
+                    CuboidBenefitModel.BenefitModel benefitModel = getBenefitPolicy().calculateBenefit(cuboid, selected);
+                    if (benefitModel != null && (currentBest == null || currentBest.getBenefit() == null
+                            || benefitModel.getBenefit() > currentBest.getBenefit())) {
+                        while (!best.compareAndSet(currentBest,
+                                new CuboidBenefitModel(getCuboidStats().getCuboidModel(cuboid), benefitModel))) {
+                            currentBest = best.get();
+                            if (benefitModel.getBenefit() <= currentBest.getBenefit()) {
+                                break;
+                            }
+                        }
+                    }
+                    counter.countDown();
+                }
+            });
+        }
+
+        try {
+            counter.await();
+        } catch (InterruptedException e) {
+        }
+        return best.get();
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/74772d78/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java
new file mode 100755
index 0000000..0c9fbae
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.greedy.GreedyAlgorithm;
+import org.junit.Test;
+
+public class GreedyAlgorithmTest extends AlgorithmTestBase {
+
+    @Test
+    public void testBPUSCalculator() {
+        BenefitPolicy benefitPolicy = new BPUSCalculator(cuboidStats);
+        GreedyAlgorithm algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+
+        List<Long> recommendList = algorithm.recommend(10);
+        System.out.println("recommendList by BPUSCalculator: " + recommendList);
+        System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+    }
+
+    @Test
+    public void testPBPUSCalculator() {
+        BenefitPolicy benefitPolicy = new PBPUSCalculator(cuboidStats);
+        GreedyAlgorithm algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+
+        List<Long> recommendList = algorithm.recommend(10);
+        System.out.println("recommendList by PBPUSCalculator:" + recommendList);
+        System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+    }
+
+    @Test
+    public void testSPBPUSCalculator() {
+        BenefitPolicy benefitPolicy = new SPBPUSCalculator(cuboidStats);
+        GreedyAlgorithm algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+
+        List<Long> recommendList = algorithm.recommend(10);
+        System.out.println("recommendList by SPBPUSCalculator:" + recommendList);
+        System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+    }
+
+}


[15/22] kylin git commit: APACHE-KYLIN-2733: Introduce optimize job for adjusting cuboid set

Posted by nj...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java
new file mode 100644
index 0000000..3e7cd3e
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class UpdateCubeInfoAfterCheckpointStep extends AbstractExecutable {
+
+    private static final Logger logger = LoggerFactory.getLogger(UpdateCubeInfoAfterCheckpointStep.class);
+
+    public UpdateCubeInfoAfterCheckpointStep() {
+        super();
+    }
+
+    @Override
+    protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+        final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
+        final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+
+        Set<Long> recommendCuboids = cube.getCuboidsRecommend();
+        try {
+            List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
+            Map<Long, Long> recommendCuboidsWithStats = CuboidStatsReaderUtil
+                    .readCuboidStatsFromSegments(recommendCuboids, newSegments);
+            if (recommendCuboidsWithStats == null) {
+                throw new RuntimeException("Fail to get statistics info for recommended cuboids after optimization!!!");
+            }
+            cubeManager.promoteCheckpointOptimizeSegments(cube, recommendCuboidsWithStats,
+                    newSegments.toArray(new CubeSegment[newSegments.size()]));
+            return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+        } catch (Exception e) {
+            logger.error("fail to update cube after build", e);
+            return new ExecuteResult(e, e.getLocalizedMessage());
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
new file mode 100644
index 0000000..965111b
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class UpdateCubeInfoAfterOptimizeStep extends AbstractExecutable {
+
+    private static final Logger logger = LoggerFactory.getLogger(UpdateCubeInfoAfterOptimizeStep.class);
+
+    public UpdateCubeInfoAfterOptimizeStep() {
+        super();
+    }
+
+    @Override
+    protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+        final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
+        final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+        final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+
+        CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
+        long sourceCount = originalSegment.getInputRecords();
+        long sourceSizeBytes = originalSegment.getInputRecordsSize();
+
+        CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
+        long cubeSizeBytes = cubingJob.findCubeSizeBytes();
+
+        segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
+        segment.setLastBuildTime(System.currentTimeMillis());
+        segment.setSizeKB(cubeSizeBytes / 1024);
+        segment.setInputRecords(sourceCount);
+        segment.setInputRecordsSize(sourceSizeBytes);
+
+        try {
+            cubeManager.promoteNewlyOptimizeSegments(cube, segment);
+            return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+        } catch (IOException e) {
+            logger.error("fail to update cube after build", e);
+            return new ExecuteResult(e, e.getLocalizedMessage());
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
new file mode 100644
index 0000000..0cd7264
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class UpdateOldCuboidShardJob extends AbstractHadoopJob {
+
+    private static final Logger logger = LoggerFactory.getLogger(UpdateOldCuboidShardJob.class);
+
+    @Override
+    public int run(String[] args) throws Exception {
+        Options options = new Options();
+        try {
+            options.addOption(OPTION_JOB_NAME);
+            options.addOption(OPTION_CUBE_NAME);
+            options.addOption(OPTION_SEGMENT_ID);
+            options.addOption(OPTION_INPUT_PATH);
+            options.addOption(OPTION_OUTPUT_PATH);
+            parseOptions(options, args);
+
+            job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+            String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+            String segmentID = getOptionValue(OPTION_SEGMENT_ID);
+            Path input = new Path(getOptionValue(OPTION_INPUT_PATH));
+            Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+
+            CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+            CubeInstance cube = cubeMgr.getCube(cubeName);
+            CubeSegment optSegment = cube.getSegmentById(segmentID);
+            CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(optSegment);
+
+            logger.info("Starting: " + job.getJobName());
+
+            setJobClasspath(job, cube.getConfig());
+
+            // Mapper
+            job.setMapperClass(UpdateOldCuboidShardMapper.class);
+
+            // Reducer
+            job.setNumReduceTasks(0);
+
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(Text.class);
+
+            // Input
+            job.setInputFormatClass(SequenceFileInputFormat.class);
+            FileInputFormat.setInputPaths(job, input);
+            // Output
+            job.setOutputFormatClass(SequenceFileOutputFormat.class);
+            FileOutputFormat.setOutputPath(job, output);
+
+            // set job configuration
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
+            // add metadata to distributed cache
+            attachSegmentsMetadataWithDict(Lists.newArrayList(optSegment, originalSegment), job.getConfiguration());
+
+            this.deletePath(job.getConfiguration(), output);
+
+            return waitForCompletion(job);
+        } catch (Exception e) {
+            logger.error("error in CuboidJob", e);
+            printUsage(options);
+            throw e;
+        } finally {
+            if (job != null)
+                cleanupTempConfFile(job.getConfiguration());
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
new file mode 100644
index 0000000..cf3c29e
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import static org.apache.kylin.engine.mr.JobBuilderSupport.PathNameCuboidBase;
+import static org.apache.kylin.engine.mr.JobBuilderSupport.PathNameCuboidOld;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.SplittedBytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.kv.RowKeyEncoder;
+import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class UpdateOldCuboidShardMapper extends KylinMapper<Text, Text, Text, Text> {
+
+    private static final Logger logger = LoggerFactory.getLogger(UpdateOldCuboidShardMapper.class);
+
+    private MultipleOutputs mos;
+    private long baseCuboid;
+
+    private CubeDesc cubeDesc;
+    private RowKeySplitter rowKeySplitter;
+    private RowKeyEncoderProvider rowKeyEncoderProvider;
+
+    private Text outputKey = new Text();
+    private byte[] newKeyBodyBuf = new byte[RowConstants.ROWKEY_BUFFER_SIZE];
+    private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
+
+    @Override
+    protected void setup(Context context) throws IOException {
+        super.bindCurrentConfiguration(context.getConfiguration());
+        mos = new MultipleOutputs(context);
+
+        String cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
+        String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
+
+        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+
+        CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
+        CubeSegment cubeSegment = cube.getSegmentById(segmentID);
+        CubeSegment oldSegment = cube.getOriginalSegmentToOptimize(cubeSegment);
+
+        cubeDesc = cube.getDescriptor();
+        baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
+
+        rowKeySplitter = new RowKeySplitter(oldSegment, 65, 256);
+        rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
+    }
+
+    @Override
+    public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
+        long cuboidID = rowKeySplitter.split(key.getBytes());
+
+        Cuboid cuboid = Cuboid.findForMandatory(cubeDesc, cuboidID);
+        int fullKeySize = buildKey(cuboid, rowKeySplitter.getSplitBuffers());
+        outputKey.set(newKeyBuf.array(), 0, fullKeySize);
+
+        String baseOutputPath = PathNameCuboidOld;
+        if (cuboidID == baseCuboid) {
+            baseOutputPath = PathNameCuboidBase;
+        }
+        mos.write(outputKey, value, generateFileName(baseOutputPath));
+    }
+
+    private int buildKey(Cuboid cuboid, SplittedBytes[] splitBuffers) {
+        RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid);
+
+        int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId
+        int endIdx = startIdx + Long.bitCount(cuboid.getId());
+        int offset = 0;
+        for (int i = startIdx; i < endIdx; i++) {
+            System.arraycopy(splitBuffers[i].value, 0, newKeyBodyBuf, offset, splitBuffers[i].length);
+            offset += splitBuffers[i].length;
+        }
+
+        int fullKeySize = rowkeyEncoder.getBytesLength();
+        while (newKeyBuf.array().length < fullKeySize) {
+            newKeyBuf.set(new byte[newKeyBuf.length() * 2]);
+        }
+        newKeyBuf.set(0, fullKeySize);
+
+        rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf);
+
+        return fullKeySize;
+    }
+
+    @Override
+    public void doCleanup(Context context) throws IOException, InterruptedException {
+        mos.close();
+
+        Path outputDirBase = new Path(context.getConfiguration().get(FileOutputFormat.OUTDIR), PathNameCuboidBase);
+        FileSystem fs = FileSystem.get(context.getConfiguration());
+        if (!fs.exists(outputDirBase)) {
+            fs.mkdirs(outputDirBase);
+            SequenceFile
+                    .createWriter(context.getConfiguration(),
+                            SequenceFile.Writer.file(new Path(outputDirBase, "part-m-00000")),
+                            SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class))
+                    .close();
+        }
+    }
+
+    private String generateFileName(String subDir) {
+        return subDir + "/part";
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
index 08ed207..81d97b4 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
@@ -20,6 +20,7 @@ package org.apache.kylin.engine.spark;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.IBatchCubingEngine;
+import org.apache.kylin.engine.mr.BatchOptimizeJobBuilder2;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 
@@ -46,6 +47,11 @@ public class SparkBatchCubingEngine implements IBatchCubingEngine {
     }
 
     @Override
+    public DefaultChainedExecutable createBatchOptimizeJob(CubeSegment optimizeSegment, String submitter) {
+        return new BatchOptimizeJobBuilder2(optimizeSegment, submitter).build();
+    }
+
+    @Override
     public Class<?> getSourceInterface() {
         return null;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index a370292..8f821dd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -39,6 +39,7 @@ import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
+import org.apache.kylin.job.exception.JobException;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.ISourceAware;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -50,6 +51,7 @@ import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobBuildRequest2;
+import org.apache.kylin.rest.request.JobOptimizeRequest;
 import org.apache.kylin.rest.response.GeneralResponse;
 import org.apache.kylin.rest.response.HBaseResponse;
 import org.apache.kylin.rest.service.CubeService;
@@ -307,6 +309,64 @@ public class CubeController extends BasicController {
         }
     }
 
+    /**
+     * Send a optimize cube job
+     *
+     * @param cubeName Cube ID
+     * @return JobInstance of CheckpointExecutable
+     */
+    @RequestMapping(value = "/{cubeName}/optimize", method = { RequestMethod.PUT })
+    @ResponseBody
+    public JobInstance optimize(@PathVariable String cubeName, @RequestBody JobOptimizeRequest jobOptimizeRequest) {
+        try {
+            String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+            CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
+
+            if (cube == null) {
+                throw new InternalErrorException("Cannot find cube " + cubeName);
+            }
+            logger.info("cuboid recommend:" + jobOptimizeRequest.getCuboidsRecommend());
+            return jobService.submitOptimizeJob(cube, jobOptimizeRequest.getCuboidsRecommend(), submitter).getFirst();
+        } catch (JobException e) {
+            logger.error(e.getLocalizedMessage(), e);
+            throw new BadRequestException(e.getLocalizedMessage());
+        } catch (Exception e) {
+            logger.error(e.getLocalizedMessage(), e);
+            throw new InternalErrorException(e.getLocalizedMessage());
+        }
+    }
+
+    /**
+     * Send a optimize cube segment job
+     *
+     * @param cubeName Cube ID
+     * @param segmentID for segment to be optimized
+     */
+    @RequestMapping(value = "/{cubeName}/recover_segment_optimize/{segmentID}", method = { RequestMethod.PUT })
+    @ResponseBody
+    public JobInstance recoverSegmentOptimize(@PathVariable String cubeName, @PathVariable String segmentID) {
+        try {
+            String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+            CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
+            if (cube == null) {
+                throw new InternalErrorException("Cannot find cube " + cubeName);
+            }
+
+            CubeSegment segment = cube.getSegmentById(segmentID);
+            if (segment == null) {
+                throw new InternalErrorException("Cannot find segment '" + segmentID + "'");
+            }
+
+            return jobService.submitRecoverSegmentOptimizeJob(segment, submitter);
+        } catch (JobException e) {
+            logger.error(e.getLocalizedMessage(), e);
+            throw new BadRequestException(e.getLocalizedMessage());
+        } catch (Exception e) {
+            logger.error(e.getLocalizedMessage(), e);
+            throw new InternalErrorException(e.getLocalizedMessage());
+        }
+    }
+
     @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT }, produces = { "application/json" })
     @ResponseBody
     public CubeInstance disableCube(@PathVariable String cubeName) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest.java
new file mode 100644
index 0000000..51e8e7c
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.request;
+
+import java.util.Set;
+
+public class JobOptimizeRequest {
+
+    private Set<Long> cuboidsRecommend;
+
+    public Set<Long> getCuboidsRecommend() {
+        return cuboidsRecommend;
+    }
+
+    public void setCuboidsRecommend(Set<Long> cuboidsRecommend) {
+        this.cuboidsRecommend = cuboidsRecommend;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 4820ccd..93af53b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -70,6 +70,8 @@ import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 
 /**
  * Stateless & lightweight service facade of cube management functions.
@@ -509,6 +511,8 @@ public class CubeService extends BasicService {
 
         CubeUpdate update = new CubeUpdate(cube);
         update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
+        update.setCuboids(Maps.<Long, Long> newHashMap());
+        update.setCuboidsRecommend(Sets.<Long> newHashSet());
         CubeManager.getInstance(getConfig()).updateCube(update);
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 432d300..a437934 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -34,11 +34,13 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.directory.api.util.Strings;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.CubeUpdate;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.BatchOptimizeJobCheckpointBuilder;
 import org.apache.kylin.engine.mr.CubingJob;
 import org.apache.kylin.engine.mr.common.JobInfoConverter;
 import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
@@ -48,6 +50,7 @@ import org.apache.kylin.job.SchedulerFactory;
 import org.apache.kylin.job.constant.JobStatusEnum;
 import org.apache.kylin.job.constant.JobTimeFilterEnum;
 import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.exception.JobException;
 import org.apache.kylin.job.exception.SchedulerException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.CheckpointExecutable;
@@ -71,6 +74,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 
 import com.google.common.base.Function;
@@ -271,6 +275,142 @@ public class JobService extends BasicService implements InitializingBean {
         return jobInstance;
     }
 
+    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
+    public Pair<JobInstance, List<JobInstance>> submitOptimizeJob(CubeInstance cube, Set<Long> cuboidsRecommend,
+            String submitter) throws IOException, JobException {
+
+        Pair<JobInstance, List<JobInstance>> result = submitOptimizeJobInternal(cube, cuboidsRecommend, submitter);
+        accessService.init(result.getFirst(), null);
+        accessService.inherit(result.getFirst(), cube);
+        for (JobInstance jobInstance : result.getSecond()) {
+            accessService.init(jobInstance, null);
+            accessService.inherit(jobInstance, cube);
+        }
+
+        return result;
+    }
+
+    private Pair<JobInstance, List<JobInstance>> submitOptimizeJobInternal(CubeInstance cube,
+            Set<Long> cuboidsRecommend, String submitter) throws IOException {
+        Message msg = MsgPicker.getMsg();
+
+        if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+            throw new BadRequestException(String.format(msg.getBUILD_BROKEN_CUBE(), cube.getName()));
+        }
+
+        checkCubeDescSignature(cube);
+        checkAllowOptimization(cube, cuboidsRecommend);
+
+        CubeSegment[] optimizeSegments = null;
+        try {
+            /** Add optimize segments */
+            optimizeSegments = getCubeManager().optimizeSegments(cube, cuboidsRecommend);
+            List<JobInstance> optimizeJobInstances = Lists.newLinkedList();
+
+            /** Add optimize jobs */
+            List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
+            for (CubeSegment optimizeSegment : optimizeSegments) {
+                DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
+                getExecutableManager().addJob(optimizeJob);
+
+                optimizeJobList.add(optimizeJob);
+                optimizeJobInstances.add(getSingleJobInstance(optimizeJob));
+            }
+
+            /** Add checkpoint job for batch jobs */
+            CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cube, submitter).build();
+            checkpointJob.addTaskListForCheck(optimizeJobList);
+
+            getExecutableManager().addJob(checkpointJob);
+
+            return new Pair(getCheckpointJobInstance(checkpointJob), optimizeJobInstances);
+        } catch (Exception e) {
+            if (optimizeSegments != null) {
+                logger.error("Job submission might failed for NEW segments {}, will clean the NEW segments from cube",
+                        optimizeSegments);
+                try {
+                    // Remove this segments
+                    CubeUpdate cubeBuilder = new CubeUpdate(cube);
+                    cubeBuilder.setToRemoveSegs(optimizeSegments);
+                    getCubeManager().updateCube(cubeBuilder);
+                } catch (Exception ee) {
+                    // swallow the exception
+                    logger.error("Clean New segments failed, ignoring it", e);
+                }
+            }
+            throw e;
+        }
+    }
+
+    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
+    public JobInstance submitRecoverSegmentOptimizeJob(CubeSegment segment, String submitter)
+            throws IOException, JobException {
+        CubeInstance cubeInstance = segment.getCubeInstance();
+
+        checkCubeDescSignature(cubeInstance);
+
+        String cubeName = cubeInstance.getName();
+        List<JobInstance> jobInstanceList = searchJobsByCubeName(cubeName, null,
+                Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.ERROR),
+                JobTimeFilterEnum.ALL, JobSearchMode.CHECKPOINT_ONLY);
+        if (jobInstanceList.size() > 1) {
+            throw new IllegalStateException("Exist more than one CheckpointExecutable for cube " + cubeName);
+        } else if (jobInstanceList.size() == 0) {
+            throw new IllegalStateException("There's no CheckpointExecutable for cube " + cubeName);
+        }
+        CheckpointExecutable checkpointExecutable = (CheckpointExecutable) getExecutableManager()
+                .getJob(jobInstanceList.get(0).getId());
+
+        AbstractExecutable toBeReplaced = null;
+        for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
+            if (taskForCheck instanceof CubingJob) {
+                CubingJob subCubingJob = (CubingJob) taskForCheck;
+                String segmentName = CubingExecutableUtil.getSegmentName(subCubingJob.getParams());
+                if (segmentName != null && segmentName.equals(segment.getName())) {
+                    String segmentID = CubingExecutableUtil.getSegmentId(subCubingJob.getParams());
+                    CubeSegment beingOptimizedSegment = cubeInstance.getSegmentById(segmentID);
+                    if (beingOptimizedSegment != null) { // beingOptimizedSegment exists & should not be recovered
+                        throw new IllegalStateException("Segment " + beingOptimizedSegment.getName() + "-"
+                                + beingOptimizedSegment.getUuid()
+                                + " still exists. Please delete it or discard the related optimize job first!!!");
+                    }
+                    toBeReplaced = taskForCheck;
+                    break;
+                }
+            }
+        }
+        if (toBeReplaced == null) {
+            throw new IllegalStateException("There's no CubingJob for segment " + segment.getName()
+                    + " in CheckpointExecutable " + checkpointExecutable.getName());
+        }
+
+        /** Add CubingJob for the related segment **/
+        CubeSegment optimizeSegment = getCubeManager().appendSegment(cubeInstance, segment.getDateRangeStart(),
+                segment.getDateRangeEnd());
+        CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
+        cubeBuilder.setToAddSegs(optimizeSegment);
+        getCubeManager().updateCube(cubeBuilder);
+
+        DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
+
+        getExecutableManager().addJob(optimizeJob);
+
+        JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);
+        accessService.init(optimizeJobInstance, null);
+        accessService.inherit(optimizeJobInstance, cubeInstance);
+
+        /** Update the checkpoint job */
+        checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
+                optimizeJob);
+
+        getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(),
+                checkpointExecutable.getSubTasksForCheck());
+
+        return optimizeJobInstance;
+    }
+
     private void checkCubeDescSignature(CubeInstance cube) {
         Message msg = MsgPicker.getMsg();
 
@@ -279,8 +419,25 @@ public class JobService extends BasicService implements InitializingBean {
                     String.format(msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
     }
 
+    private void checkAllowOptimization(CubeInstance cube, Set<Long> cuboidsRecommend) {
+        long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
+        if (!cuboidsRecommend.contains(baseCuboid)) {
+            throw new BadRequestException("The recommend cuboids should contain the base cuboid " + baseCuboid);
+        }
+        Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+        if (currentCuboidSet.equals(cuboidsRecommend)) {
+            throw new BadRequestException(
+                    "The recommend cuboids are the same as the current cuboids. It's no need to do optimization.");
+        }
+    }
+
     public JobInstance getJobInstance(String uuid) {
-        return getSingleJobInstance(getExecutableManager().getJob(uuid));
+        AbstractExecutable job = getExecutableManager().getJob(uuid);
+        if (job instanceof CheckpointExecutable) {
+            return getCheckpointJobInstance(job);
+        } else {
+            return getSingleJobInstance(job);
+        }
     }
 
     public Output getOutput(String id) {
@@ -362,21 +519,88 @@ public class JobService extends BasicService implements InitializingBean {
             getExecutableManager().discardJob(job.getId());
             return job;
         }
-        CubeInstance cubeInstance = getCubeManager().getCube(job.getRelatedCube());
+
+        logger.info("Cancel job [" + job.getId() + "] trigger by "
+                + SecurityContextHolder.getContext().getAuthentication().getName());
+        if (job.getStatus() == JobStatusEnum.FINISHED) {
+            throw new IllegalStateException(
+                    "The job " + job.getId() + " has already been finished and cannot be discarded.");
+        }
+        if (job.getStatus() == JobStatusEnum.DISCARDED) {
+            return job;
+        }
+
+        AbstractExecutable executable = getExecutableManager().getJob(job.getId());
+        if (executable instanceof CubingJob) {
+            cancelCubingJobInner((CubingJob) executable);
+        } else if (executable instanceof CheckpointExecutable) {
+            cancelCheckpointJobInner((CheckpointExecutable) executable);
+        } else {
+            getExecutableManager().discardJob(executable.getId());
+        }
+        return job;
+    }
+
+    private void cancelCubingJobInner(CubingJob cubingJob) throws IOException {
+        CubeInstance cubeInstance = getCubeManager().getCube(CubingExecutableUtil.getCubeName(cubingJob.getParams()));
         // might not a cube job
-        final String segmentIds = job.getRelatedSegment();
-        for (String segmentId : StringUtils.split(segmentIds)) {
-            final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
-            if (segment != null && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getDateRangeEnd() == 0)) {
-                // Remove this segments
+        final String segmentIds = CubingExecutableUtil.getSegmentId(cubingJob.getParams());
+        if (!StringUtils.isEmpty(segmentIds)) {
+            List<CubeSegment> toRemoveSegments = Lists.newLinkedList();
+            for (String segmentId : StringUtils.split(segmentIds)) {
+                final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
+                if (segment != null && segment.getStatus() != SegmentStatusEnum.READY) {
+                    toRemoveSegments.add(segment);
+                }
+            }
+            if (!toRemoveSegments.isEmpty()) {
                 CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
-                cubeBuilder.setToRemoveSegs(segment);
+                cubeBuilder.setToRemoveSegs(toRemoveSegments.toArray(new CubeSegment[toRemoveSegments.size()]));
                 getCubeManager().updateCube(cubeBuilder);
             }
         }
-        getExecutableManager().discardJob(job.getId());
+        getExecutableManager().discardJob(cubingJob.getId());
+    }
 
-        return job;
+    private void cancelCheckpointJobInner(CheckpointExecutable checkpointExecutable) throws IOException {
+        List<String> segmentIdList = Lists.newLinkedList();
+        List<String> jobIdList = Lists.newLinkedList();
+        jobIdList.add(checkpointExecutable.getId());
+        setRelatedIdList(checkpointExecutable, segmentIdList, jobIdList);
+
+        CubeInstance cubeInstance = getCubeManager()
+                .getCube(CubingExecutableUtil.getCubeName(checkpointExecutable.getParams()));
+        if (!segmentIdList.isEmpty()) {
+            List<CubeSegment> toRemoveSegments = Lists.newLinkedList();
+            for (String segmentId : segmentIdList) {
+                final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
+                if (segment != null && segment.getStatus() != SegmentStatusEnum.READY) {
+                    toRemoveSegments.add(segment);
+                }
+            }
+
+            CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
+            cubeBuilder.setToRemoveSegs(toRemoveSegments.toArray(new CubeSegment[toRemoveSegments.size()]));
+            cubeBuilder.setCuboidsRecommend(Sets.<Long> newHashSet()); //Set recommend cuboids to be null
+            getCubeManager().updateCube(cubeBuilder);
+        }
+
+        for (String jobId : jobIdList) {
+            getExecutableManager().discardJob(jobId);
+        }
+    }
+
+    private void setRelatedIdList(CheckpointExecutable checkpointExecutable, List<String> segmentIdList,
+            List<String> jobIdList) {
+        for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
+            jobIdList.add(taskForCheck.getId());
+            if (taskForCheck instanceof CubingJob) {
+                segmentIdList.addAll(Lists
+                        .newArrayList(StringUtils.split(CubingExecutableUtil.getSegmentId(taskForCheck.getParams()))));
+            } else if (taskForCheck instanceof CheckpointExecutable) {
+                setRelatedIdList((CheckpointExecutable) taskForCheck, segmentIdList, jobIdList);
+            }
+        }
     }
 
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index fc52701..f2e2ddd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -24,6 +24,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.conf.Configuration;
@@ -66,6 +67,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
     CubeInstance cube = null;
     CubeDesc cubeDesc = null;
     String segmentID = null;
+    String cuboidModeName = null;
     KylinConfig kylinConfig;
     Path partitionFilePath;
 
@@ -77,6 +79,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
         options.addOption(OPTION_SEGMENT_ID);
         options.addOption(OPTION_PARTITION_FILE_PATH);
         options.addOption(OPTION_STATISTICS_ENABLED);
+        options.addOption(OPTION_CUBOID_MODE);
         parseOptions(options, args);
 
         partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
@@ -88,13 +91,27 @@ public class CreateHTableJob extends AbstractHadoopJob {
         cubeDesc = cube.getDescriptor();
         kylinConfig = cube.getConfig();
         segmentID = getOptionValue(OPTION_SEGMENT_ID);
+        cuboidModeName = getOptionValue(OPTION_CUBOID_MODE);
         CubeSegment cubeSegment = cube.getSegmentById(segmentID);
 
         Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
 
         byte[][] splitKeys;
         if (statsEnabled) {
-            final Map<Long, Double> cuboidSizeMap = new CubeStatsReader(cubeSegment, kylinConfig).getCuboidSizeMap();
+            Map<Long, Double> cuboidSizeMap = new CubeStatsReader(cubeSegment, null, kylinConfig).getCuboidSizeMap();
+            Set<Long> buildingCuboids = cube.getCuboidsByMode(cuboidModeName);
+            if (buildingCuboids != null && !buildingCuboids.isEmpty()) {
+                Map<Long, Double> optimizedCuboidSizeMap = Maps.newHashMapWithExpectedSize(buildingCuboids.size());
+                for (Long cuboid : buildingCuboids) {
+                    Double cuboidSize = cuboidSizeMap.get(cuboid);
+                    if (cuboidSize == null) {
+                        logger.warn(cuboid + "cuboid's size is null will replace by 0");
+                        cuboidSize = 0.0;
+                    }
+                    optimizedCuboidSizeMap.put(cuboid, cuboidSize);
+                }
+                cuboidSizeMap = optimizedCuboidSizeMap;
+            }
             splitKeys = getRegionSplitsFromCuboidStatistics(cuboidSizeMap, kylinConfig, cubeSegment, partitionFilePath.getParent());
         } else {
             splitKeys = getRegionSplits(conf, partitionFilePath);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 31cb189..e1cf7e0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -31,13 +31,15 @@ import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.kylin.common.util.HadoopUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.engine.mr.IMROutput2;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.MapReduceUtil;
 import org.apache.kylin.engine.mr.steps.HiveToBaseCuboidMapper;
 import org.apache.kylin.engine.mr.steps.InMemCuboidMapper;
 import org.apache.kylin.engine.mr.steps.MergeCuboidJob;
 import org.apache.kylin.engine.mr.steps.NDCuboidMapper;
-import org.apache.kylin.engine.mr.steps.ReducerNumSizing;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -93,13 +95,15 @@ public class HBaseMROutput2Transition implements IMROutput2 {
         }
 
         @Override
-        public void configureJobOutput(Job job, String output, CubeSegment segment, int level) throws Exception {
+        public void configureJobOutput(Job job, String output, CubeSegment segment, CuboidScheduler cuboidScheduler,
+                int level) throws Exception {
             int reducerNum = 1;
             Class mapperClass = job.getMapperClass();
             if (mapperClass == HiveToBaseCuboidMapper.class || mapperClass == NDCuboidMapper.class) {
-                reducerNum = ReducerNumSizing.getLayeredCubingReduceTaskNum(segment, AbstractHadoopJob.getTotalMapInputMB(job), level);
+                reducerNum = MapReduceUtil.getLayeredCubingReduceTaskNum(segment, cuboidScheduler,
+                        AbstractHadoopJob.getTotalMapInputMB(job), level);
             } else if (mapperClass == InMemCuboidMapper.class) {
-                reducerNum = ReducerNumSizing.getInmemCubingReduceTaskNum(segment);
+                reducerNum = MapReduceUtil.getInmemCubingReduceTaskNum(segment, cuboidScheduler);
             }
             Path outputPath = new Path(output);
             FileOutputFormat.setOutputPath(job, outputPath);
@@ -149,7 +153,8 @@ public class HBaseMROutput2Transition implements IMROutput2 {
 
         @Override
         public void configureJobOutput(Job job, String output, CubeSegment segment) throws Exception {
-            int reducerNum = ReducerNumSizing.getLayeredCubingReduceTaskNum(segment, AbstractHadoopJob.getTotalMapInputMB(job), -1);
+            int reducerNum = MapReduceUtil.getLayeredCubingReduceTaskNum(segment, segment.getCuboidScheduler(),
+                    AbstractHadoopJob.getTotalMapInputMB(job), -1);
             job.setNumReduceTasks(reducerNum);
 
             Path outputPath = new Path(output);
@@ -185,4 +190,26 @@ public class HBaseMROutput2Transition implements IMROutput2 {
             throw new IllegalStateException("No merging segment's last build job ID equals " + jobID);
         }
     }
+
+    public IMRBatchOptimizeOutputSide2 getBatchOptimizeOutputSide(final CubeSegment seg) {
+        return new IMRBatchOptimizeOutputSide2() {
+            HBaseMRSteps steps = new HBaseMRSteps(seg);
+
+            @Override
+            public void addStepPhase1_CreateHTable(DefaultChainedExecutable jobFlow) {
+                jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId(), CuboidModeEnum.RECOMMEND));
+            }
+
+            @Override
+            public void addStepPhase2_BuildCube(DefaultChainedExecutable jobFlow) {
+                jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId()));
+                jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId()));
+            }
+
+            @Override
+            public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow) {
+                steps.addOptimizeGarbageCollectionSteps(jobFlow);
+            }
+        };
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index 6f69e8c..41e80e3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -24,6 +24,7 @@ import java.util.List;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
 import org.apache.kylin.engine.mr.CubingJob;
 import org.apache.kylin.engine.mr.JobBuilderSupport;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
@@ -32,6 +33,7 @@ import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
 import org.apache.kylin.engine.mr.common.MapReduceExecutable;
 import org.apache.kylin.job.constant.ExecutableConstants;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 
 import com.google.common.base.Preconditions;
@@ -82,7 +84,15 @@ public class HBaseMRSteps extends JobBuilderSupport {
         return createCreateHTableStep(jobId, true);
     }
 
+    public HadoopShellExecutable createCreateHTableStepWithStats(String jobId, CuboidModeEnum cuboidMode) {
+        return createCreateHTableStep(jobId, true, cuboidMode);
+    }
+
     private HadoopShellExecutable createCreateHTableStep(String jobId, boolean withStats) {
+        return createCreateHTableStep(jobId, withStats, CuboidModeEnum.CURRENT);
+    }
+
+    private HadoopShellExecutable createCreateHTableStep(String jobId, boolean withStats, CuboidModeEnum cuboidMode) {
         HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
         createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
         StringBuilder cmd = new StringBuilder();
@@ -90,6 +100,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
         appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
         appendExecCmdParameters(cmd, BatchConstants.ARG_PARTITION, getRowkeyDistributionOutputPath(jobId) + "/part-r-00000");
         appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_ENABLED, String.valueOf(withStats));
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());
 
         createHtableStep.setJobParams(cmd.toString());
         createHtableStep.setJobClass(CreateHTableJob.class);
@@ -167,6 +178,35 @@ public class HBaseMRSteps extends JobBuilderSupport {
         return result;
     }
 
+    public MergeGCStep createOptimizeGCStep() {
+        MergeGCStep result = new MergeGCStep();
+        result.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+        result.setOldHTables(getOptimizeHTables());
+        return result;
+    }
+
+    public List<CubeSegment> getOptimizeSegments() {
+        CubeInstance cube = (CubeInstance) seg.getRealization();
+        List<CubeSegment> newSegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY_PENDING));
+        List<CubeSegment> oldSegments = Lists.newArrayListWithExpectedSize(newSegments.size());
+        for (CubeSegment segment : newSegments) {
+            oldSegments.add(cube.getOriginalSegmentToOptimize(segment));
+        }
+        return oldSegments;
+    }
+
+    public List<String> getOptimizeHTables() {
+        return getOldHTables(getOptimizeSegments());
+    }
+
+    public List<String> getOldHTables(final List<CubeSegment> oldSegments) {
+        final List<String> oldHTables = Lists.newArrayListWithExpectedSize(oldSegments.size());
+        for (CubeSegment segment : oldSegments) {
+            oldHTables.add(segment.getStorageLocationIdentifier());
+        }
+        return oldHTables;
+    }
+
     public List<String> getMergingHTables() {
         final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
         Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + seg);
@@ -187,6 +227,18 @@ public class HBaseMRSteps extends JobBuilderSupport {
         return mergingHDFSPaths;
     }
 
+    public List<String> getOptimizeHDFSPaths() {
+        return getOldHDFSPaths(getOptimizeSegments());
+    }
+
+    public List<String> getOldHDFSPaths(final List<CubeSegment> oldSegments) {
+        final List<String> oldHDFSPaths = Lists.newArrayListWithExpectedSize(oldSegments.size());
+        for (CubeSegment oldSegment : oldSegments) {
+            oldHDFSPaths.add(getJobWorkingDir(oldSegment.getLastBuildJobID()));
+        }
+        return oldHDFSPaths;
+    }
+
     public String getHFilePath(String jobId) {
         return HBaseConnection.makeQualifiedPathInHBaseCluster(getJobWorkingDir(jobId) + "/" + seg.getRealization().getName() + "/hfile/");
     }
@@ -195,6 +247,22 @@ public class HBaseMRSteps extends JobBuilderSupport {
         return HBaseConnection.makeQualifiedPathInHBaseCluster(getJobWorkingDir(jobId) + "/" + seg.getRealization().getName() + "/rowkey_stats");
     }
 
+    public void addOptimizeGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
+        String jobId = jobFlow.getId();
+
+        jobFlow.addTask(createOptimizeGCStep());
+
+        List<String> toDeletePaths = new ArrayList<>();
+        toDeletePaths.addAll(getOptimizeHDFSPaths());
+
+        HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep();
+        step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HDFS);
+        step.setDeletePaths(toDeletePaths);
+        step.setJobId(jobId);
+
+        jobFlow.addTask(step);
+    }
+
     public void addMergingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
         String jobId = jobFlow.getId();
 


[21/22] kylin git commit: APACHE-KYLIN-2734: backend support for hot cuboids export & import

Posted by nj...@apache.org.
APACHE-KYLIN-2734: backend support for hot cuboids export & import


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/671cddd9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/671cddd9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/671cddd9

Branch: refs/heads/yaho-cube-planner
Commit: 671cddd9c26efbf9978741445981e26db7388b2e
Parents: faf0862
Author: Zhong <nj...@apache.org>
Authored: Wed Aug 30 14:51:19 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:05 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/model/CubeDesc.java   | 38 ++++++++
 .../kylin/rest/controller/CubeController.java   | 99 ++++++++++++++++++++
 .../apache/kylin/rest/service/BasicService.java |  4 +
 .../apache/kylin/rest/service/CubeService.java  | 27 ++++++
 .../apache/kylin/rest/service/QueryService.java | 15 ++-
 5 files changed, 180 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/671cddd9/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 510a7d5..7eefcd6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -168,6 +168,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
     @JsonProperty("override_kylin_properties")
     private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>();
 
+    @JsonProperty("mandatory_dimension_set_list")
+    private List<Set<String>> mandatoryDimensionSetList = Collections.emptyList();
+
     private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
     private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
     private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
@@ -438,6 +441,14 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         this.overrideKylinProps = overrideKylinProps;
     }
 
+    public List<Set<String>> getMandatoryDimensionSetList() {
+        return mandatoryDimensionSetList;
+    }
+
+    public void setMandatoryDimensionSetList(List<Set<String>> mandatoryDimensionSetList) {
+        this.mandatoryDimensionSetList = mandatoryDimensionSetList;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o)
@@ -526,6 +537,13 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
                     .append(JsonUtil.writeValueAsString(this.engineType)).append("|")//
                     .append(JsonUtil.writeValueAsString(this.storageType)).append("|");
 
+            if (mandatoryDimensionSetList != null && !mandatoryDimensionSetList.isEmpty()) {
+                for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) {
+                    TreeSet<String> sortedSet = Sets.newTreeSet(mandatoryDimensionSet);
+                    sigString.append(JsonUtil.writeValueAsString(sortedSet)).append("|");
+                }
+            }
+
             String signatureInput = sigString.toString().replaceAll("\\s+", "").toLowerCase();
 
             byte[] signature = md.digest(signatureInput.getBytes());
@@ -599,6 +617,26 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
 
         initDictionaryDesc();
         amendAllColumns();
+
+        // check if mandatory dimension set list is valid
+        validateMandatoryDimensionSetList();
+    }
+
+    public void validateMandatoryDimensionSetList() {
+        Set<String> rowKeyColumns = Sets.newHashSet();
+        for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) {
+            rowKeyColumns.add(entry.getColumn());
+        }
+
+        for (Set<String> mandatoryDimensionSet : this.mandatoryDimensionSetList) {
+            for (String columnName : mandatoryDimensionSet) {
+                if (!rowKeyColumns.contains(columnName)) {
+                    logger.info("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist");
+                    throw new IllegalStateException(
+                            "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist");
+                }
+            }
+        }
     }
 
     public CuboidScheduler getInitialCuboidScheduler() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/671cddd9/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8f821dd..c415212 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -21,13 +21,18 @@ package org.apache.kylin.rest.controller;
 import static org.apache.kylin.rest.service.CubeService.VALID_CUBENAME;
 
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 
+import javax.servlet.http.HttpServletResponse;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
@@ -35,6 +40,7 @@ import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.job.JobInstance;
@@ -44,6 +50,8 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.ISourceAware;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metrics.MetricsManager;
+import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
@@ -52,11 +60,13 @@ import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobBuildRequest2;
 import org.apache.kylin.rest.request.JobOptimizeRequest;
+import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.GeneralResponse;
 import org.apache.kylin.rest.response.HBaseResponse;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;
 import org.apache.kylin.rest.service.ProjectService;
+import org.apache.kylin.rest.service.QueryService;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -78,6 +88,7 @@ import com.fasterxml.jackson.databind.JsonMappingException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 
 /**
  * CubeController is defined as Restful API entrance for UI.
@@ -99,6 +110,10 @@ public class CubeController extends BasicController {
     @Qualifier("projectService")
     private ProjectService projectService;
 
+    @Autowired
+    @Qualifier("queryService")
+    private QueryService queryService;
+
     @RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/json" })
     @ResponseBody
     public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
@@ -707,6 +722,90 @@ public class CubeController extends BasicController {
 
     }
 
+    @RequestMapping(value = "/{cubeName}/cuboids/export", method = RequestMethod.GET)
+    @ResponseBody
+    public void cuboidsExport(@PathVariable String cubeName, @RequestParam(value = "top") Integer top,
+            HttpServletResponse response) throws IOException {
+        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+        if (cube == null) {
+            logger.error("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+            throw new BadRequestException("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+        }
+        Map<Long, Long> cuboidList = getRecommendCuboidList(cube);
+        if (cuboidList == null || cuboidList.isEmpty()) {
+            logger.warn("Cannot get recommend cuboid list for cube " + cubeName);
+        }
+        if (cuboidList.size() < top) {
+            logger.info("Only recommend " + cuboidList.size() + " cuboids less than topn " + top);
+        }
+        Iterator<Long> cuboidIterator = cuboidList.keySet().iterator();
+        RowKeyColDesc[] rowKeyColDescList = cube.getDescriptor().getRowkey().getRowKeyColumns();
+
+        List<Set<String>> dimensionSetList = Lists.newLinkedList();
+        while (top-- > 0 && cuboidIterator.hasNext()) {
+            Set<String> dimensionSet = Sets.newHashSet();
+            dimensionSetList.add(dimensionSet);
+            long cuboid = cuboidIterator.next();
+            for (int i = 0; i < rowKeyColDescList.length; i++) {
+                if ((cuboid & (1L << rowKeyColDescList[i].getBitIndex())) > 0) {
+                    dimensionSet.add(rowKeyColDescList[i].getColumn());
+                }
+            }
+        }
+
+        response.setContentType("text/json;charset=utf-8");
+        response.setHeader("Content-Disposition", "attachment; filename=\"" + cubeName + ".json\"");
+        try (PrintWriter writer = response.getWriter()) {
+            writer.write(JsonUtil.writeValueAsString(dimensionSetList));
+        } catch (IOException e) {
+            logger.error("", e);
+            throw new InternalErrorException("Failed to write: " + e.getLocalizedMessage());
+        }
+    }
+
+    private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException {
+        // Get cuboid source info
+        Map<Long, Long> optimizeHitFrequencyMap = getCuboidHitFrequency(cube.getName(), true);
+        Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName());
+        return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap);
+    }
+
+    private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
+        SQLRequest sqlRequest = new SQLRequest();
+        sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+        String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+        if (!isCuboidSource) {
+            cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+        }
+        String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+        String table = cubeService.getMetricsManager()
+                .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
+        String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ") " //
+                + "from " + table//
+                + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+                + "group by " + cuboidColumn;
+        sqlRequest.setSql(sql);
+        List<List<String>> orgHitFrequency = queryService.queryWithoutSecure(sqlRequest).getResults();
+        return cubeService.formatQueryCount(orgHitFrequency);
+    }
+
+    private Map<Long, Map<Long, Long>> getCuboidRollingUpCount(String cubeName) {
+        SQLRequest sqlRequest = new SQLRequest();
+        sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+        String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+        String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+        String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
+        String table = cubeService.getMetricsManager()
+                .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
+        String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*) " //
+                + "from " + table //
+                + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+                + "group by " + cuboidSource + ", " + cuboidTarget;
+        sqlRequest.setSql(sql);
+        List<List<String>> orgRollingUpCount = queryService.queryWithoutSecure(sqlRequest).getResults();
+        return cubeService.formatRollingUpCount(orgRollingUpCount);
+    }
+
     /**
      * Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
      * source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.

http://git-wip-us.apache.org/repos/asf/kylin/blob/671cddd9/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 66e7cfb..0c1054c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -29,6 +29,7 @@ import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
 import org.apache.kylin.metadata.draft.DraftManager;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.streaming.StreamingManager;
+import org.apache.kylin.metrics.MetricsManager;
 import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.storage.hybrid.HybridManager;
 
@@ -84,4 +85,7 @@ public abstract class BasicService {
         return DraftManager.getInstance(getConfig());
     }
 
+    public MetricsManager getMetricsManager() {
+        return MetricsManager.getInstance();
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/671cddd9/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 93af53b..fdf7bec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -24,6 +24,7 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Map;
 import java.util.WeakHashMap;
 
 import org.apache.commons.lang.StringUtils;
@@ -38,6 +39,7 @@ import org.apache.kylin.cube.cuboid.CuboidCLI;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.CuboidRecommenderUtil;
 import org.apache.kylin.job.exception.JobException;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
@@ -746,4 +748,29 @@ public class CubeService extends BasicService {
 
         return filtered;
     }
+
+    /** cube planner services */
+    public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) {
+        Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap();
+        for (List<String> hit : orgQueryCount) {
+            formattedQueryCount.put(Long.parseLong(hit.get(0)), (long) Double.parseDouble(hit.get(1)));
+        }
+        return formattedQueryCount;
+    }
+
+    public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>> orgRollingUpCount) {
+        Map<Long, Map<Long, Long>> formattedRollingUpCount = Maps.newLinkedHashMap();
+        for (List<String> rollingUp : orgRollingUpCount) {
+            Map<Long, Long> childMap = Maps.newLinkedHashMap();
+            childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2)));
+            formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)), childMap);
+        }
+        return formattedRollingUpCount;
+    }
+
+    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')")
+    public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+            Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+        return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/671cddd9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94b6196..8c767aa 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -342,7 +342,15 @@ public class QueryService extends BasicService {
         }
     }
 
+    public SQLResponse queryWithoutSecure(SQLRequest sqlRequest) {
+        return doQueryWithCache(sqlRequest, false);
+    }
+
     public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
+        return doQueryWithCache(sqlRequest, true);
+    }
+
+    public SQLResponse doQueryWithCache(SQLRequest sqlRequest, boolean secureEnabled) {
         Message msg = MsgPicker.getMsg();
 
         KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -415,7 +423,7 @@ public class QueryService extends BasicService {
                     sqlResponse.setTotalScanBytes(0);
                 }
 
-                checkQueryAuth(sqlResponse, project);
+                checkQueryAuth(sqlResponse, project, secureEnabled);
 
             } catch (Throwable e) { // calcite may throw AssertError
                 logger.error("Exception when execute sql", e);
@@ -468,8 +476,9 @@ public class QueryService extends BasicService {
         return response;
     }
 
-    protected void checkQueryAuth(SQLResponse sqlResponse, String project) throws AccessDeniedException {
-        if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
+    protected void checkQueryAuth(SQLResponse sqlResponse, String project, boolean secureEnabled)
+            throws AccessDeniedException {
+        if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled() && secureEnabled) {
             checkAuthorization(sqlResponse, project);
         }
     }


[16/22] kylin git commit: APACHE-KYLIN-2733: Introduce optimize job for adjusting cuboid set

Posted by nj...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceUtil.java
new file mode 100644
index 0000000..0379f64
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceUtil.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.job.exception.JobException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MapReduceUtil {
+
+    private static final Logger logger = LoggerFactory.getLogger(MapReduceUtil.class);
+
+    /**
+     * @param cuboidScheduler specified can provide more flexibility
+     * */
+    public static int getLayeredCubingReduceTaskNum(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler,
+            double totalMapInputMB, int level)
+            throws ClassNotFoundException, IOException, InterruptedException, JobException {
+        CubeDesc cubeDesc = cubeSegment.getCubeDesc();
+        KylinConfig kylinConfig = cubeDesc.getConfig();
+
+        double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
+        double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
+        logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio + ", level "
+                + level);
+
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, cuboidScheduler, kylinConfig);
+
+        double parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst;
+
+        if (level == -1) {
+            //merge case
+            double estimatedSize = cubeStatsReader.estimateCubeSize();
+            adjustedCurrentLayerSizeEst = estimatedSize > totalMapInputMB ? totalMapInputMB : estimatedSize;
+            logger.debug("estimated size {}, input size {}, adjustedCurrentLayerSizeEst: {}", estimatedSize,
+                    totalMapInputMB, adjustedCurrentLayerSizeEst);
+        } else if (level == 0) {
+            //base cuboid case TODO: the estimation could be very WRONG because it has no correction
+            adjustedCurrentLayerSizeEst = cubeStatsReader.estimateLayerSize(0);
+            logger.debug("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
+        } else {
+            parentLayerSizeEst = cubeStatsReader.estimateLayerSize(level - 1);
+            currentLayerSizeEst = cubeStatsReader.estimateLayerSize(level);
+            adjustedCurrentLayerSizeEst = totalMapInputMB / parentLayerSizeEst * currentLayerSizeEst;
+            logger.debug(
+                    "totalMapInputMB: {}, parentLayerSizeEst: {}, currentLayerSizeEst: {}, adjustedCurrentLayerSizeEst: {}",
+                    totalMapInputMB, parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst);
+        }
+
+        // number of reduce tasks
+        int numReduceTasks = (int) Math.round(adjustedCurrentLayerSizeEst / perReduceInputMB * reduceCountRatio + 0.99);
+
+        // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
+        if (cubeDesc.hasMemoryHungryMeasures()) {
+            logger.debug("Multiply reducer num by 4 to boost performance for memory hungry measures");
+            numReduceTasks = numReduceTasks * 4;
+        }
+
+        // at least 1 reducer by default
+        numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
+        // no more than 500 reducer by default
+        numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
+
+        return numReduceTasks;
+    }
+
+    public static int getInmemCubingReduceTaskNum(CubeSegment cubeSeg, CuboidScheduler cuboidScheduler)
+            throws IOException {
+        KylinConfig kylinConfig = cubeSeg.getConfig();
+
+        Map<Long, Double> cubeSizeMap = new CubeStatsReader(cubeSeg, cuboidScheduler, kylinConfig).getCuboidSizeMap();
+        double totalSizeInM = 0;
+        for (Double cuboidSize : cubeSizeMap.values()) {
+            totalSizeInM += cuboidSize;
+        }
+
+        double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
+        double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
+
+        // number of reduce tasks
+        int numReduceTasks = (int) Math.round(totalSizeInM / perReduceInputMB * reduceCountRatio);
+
+        // at least 1 reducer by default
+        numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
+        // no more than 500 reducer by default
+        numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
+
+        logger.info("Having total map input MB " + Math.round(totalSizeInM));
+        logger.info("Having per reduce MB " + perReduceInputMB);
+        logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + numReduceTasks);
+        return numReduceTasks;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
index cfac0e7..3b68bbf 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
@@ -38,7 +38,7 @@ public class StatisticsDecisionUtil {
     protected static final Logger logger = LoggerFactory.getLogger(StatisticsDecisionUtil.class);
 
     public static void decideCubingAlgorithm(CubingJob cubingJob, CubeSegment seg) throws IOException {
-        CubeStatsReader cubeStats = new CubeStatsReader(seg, seg.getConfig());
+        CubeStatsReader cubeStats = new CubeStatsReader(seg, null, seg.getConfig());
         decideCubingAlgorithm(cubingJob, seg, cubeStats.getMapperOverlapRatioOfFirstBuild(),
                 cubeStats.getMapperNumberOfFirstBuild());
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 93e413b..229bd85 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -60,7 +60,7 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
     @Override
     protected void setup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
-        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
+        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
         segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
         final KylinConfig kylinConfig = AbstractHadoopJob.loadKylinPropsAndMetadata();
         cube = CubeManager.getInstance(kylinConfig).getCube(cubeName);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
new file mode 100644
index 0000000..b60076c
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CalculateStatsFromBaseCuboidJob extends AbstractHadoopJob {
+
+    private static final Logger logger = LoggerFactory.getLogger(CalculateStatsFromBaseCuboidJob.class);
+
+    @Override
+    public int run(String[] args) throws Exception {
+        Options options = new Options();
+
+        try {
+            options.addOption(OPTION_JOB_NAME);
+            options.addOption(OPTION_CUBE_NAME);
+            options.addOption(OPTION_SEGMENT_ID);
+            options.addOption(OPTION_INPUT_PATH);
+            options.addOption(OPTION_OUTPUT_PATH);
+            options.addOption(OPTION_STATISTICS_SAMPLING_PERCENT);
+            options.addOption(OPTION_CUBOID_MODE);
+            parseOptions(options, args);
+
+            job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+            String cubeName = getOptionValue(OPTION_CUBE_NAME);
+            String segmentID = getOptionValue(OPTION_SEGMENT_ID);
+            Path input = new Path(getOptionValue(OPTION_INPUT_PATH));
+            Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+            String statistics_sampling_percent = getOptionValue(OPTION_STATISTICS_SAMPLING_PERCENT);
+            String cuboidMode = getOptionValue(OPTION_CUBOID_MODE);
+
+            CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+            CubeInstance cube = cubeMgr.getCube(cubeName);
+            CubeSegment cubeSegment = cube.getSegmentById(segmentID);
+
+            job.getConfiguration().set(BatchConstants.CFG_CUBOID_MODE, cuboidMode);
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
+            job.getConfiguration().set(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, statistics_sampling_percent);
+            logger.info("Starting: " + job.getJobName());
+
+            setJobClasspath(job, cube.getConfig());
+
+            setupMapper(input);
+            setupReducer(output, 1);
+
+            attachSegmentMetadataWithDict(cubeSegment, job.getConfiguration());
+
+            return waitForCompletion(job);
+
+        } catch (Exception e) {
+            logger.error("error in CalculateStatsFromBaseCuboidJob", e);
+            printUsage(options);
+            throw e;
+        } finally {
+            if (job != null)
+                cleanupTempConfFile(job.getConfiguration());
+        }
+    }
+
+    private void setupMapper(Path input) throws IOException {
+        FileInputFormat.setInputPaths(job, input);
+        job.setMapperClass(CalculateStatsFromBaseCuboidMapper.class);
+        job.setInputFormatClass(SequenceFileInputFormat.class);
+        job.setMapOutputKeyClass(Text.class);
+        job.setMapOutputValueClass(Text.class);
+    }
+
+    private void setupReducer(Path output, int numberOfReducers) throws IOException {
+        job.setReducerClass(CalculateStatsFromBaseCuboidReducer.class);
+        job.setOutputFormatClass(SequenceFileOutputFormat.class);
+        job.setOutputKeyClass(NullWritable.class);
+        job.setOutputValueClass(Text.class);
+        job.setNumReduceTasks(numberOfReducers);
+
+        FileOutputFormat.setOutputPath(job, output);
+        job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
+
+        deletePath(job.getConfiguration(), output);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
new file mode 100644
index 0000000..fbc7c7c
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidUtil;
+import org.apache.kylin.cube.kv.RowKeyDecoder;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
+public class CalculateStatsFromBaseCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
+    private static final Logger logger = LoggerFactory.getLogger(CalculateStatsFromBaseCuboidMapper.class);
+
+    protected int nRowKey;
+    protected long baseCuboidId;
+
+    private int samplingPercentage;
+    private int rowCount = 0;
+
+    private HLLCounter[] allCuboidsHLL = null;
+    private Long[] cuboidIds;
+    private Integer[][] allCuboidsBitSet = null;
+    private ByteArray[] row_hashcodes = null;
+    private HashFunction hf = null;
+
+    RowKeyDecoder rowKeyDecoder;
+
+    protected Text outputKey = new Text();
+    protected Text outputValue = new Text();
+
+    @Override
+    protected void setup(Context context) throws IOException {
+        Configuration conf = context.getConfiguration();
+        HadoopUtil.setCurrentConfiguration(conf);
+        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+
+        String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+        CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
+        CubeDesc cubeDesc = cube.getDescriptor();
+        CubeSegment cubeSegment = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID));
+
+        baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId();
+        nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
+
+        String cuboidModeName = conf.get(BatchConstants.CFG_CUBOID_MODE);
+        Set<Long> cuboidIdSet = cube.getCuboidsByMode(cuboidModeName);
+
+        cuboidIds = cuboidIdSet.toArray(new Long[cuboidIdSet.size()]);
+        allCuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, nRowKey);
+
+        samplingPercentage = Integer
+                .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
+
+        allCuboidsHLL = new HLLCounter[cuboidIds.length];
+        for (int i = 0; i < cuboidIds.length; i++) {
+            allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision());
+        }
+
+        hf = Hashing.murmur3_32();
+        row_hashcodes = new ByteArray[nRowKey];
+        for (int i = 0; i < nRowKey; i++) {
+            row_hashcodes[i] = new ByteArray();
+        }
+
+        rowKeyDecoder = new RowKeyDecoder(cubeSegment);
+    }
+
+    @Override
+    public void doMap(Text key, Text value, Context context) throws InterruptedException, IOException {
+        long cuboidID = rowKeyDecoder.decode(key.getBytes());
+        if (cuboidID != baseCuboidId) {
+            return; // Skip data from cuboids which are not the base cuboid
+        }
+
+        List<String> keyValues = rowKeyDecoder.getValues();
+
+        if (rowCount < samplingPercentage) {
+            Preconditions.checkArgument(nRowKey == keyValues.size());
+
+            String[] row = keyValues.toArray(new String[keyValues.size()]);
+
+            putRowKeyToHLL(row);
+        }
+
+        if (++rowCount == 100)
+            rowCount = 0;
+    }
+
+    public void putRowKeyToHLL(String[] row) {
+        //generate hash for each row key column
+        for (int i = 0; i < nRowKey; i++) {
+            Hasher hc = hf.newHasher();
+            String colValue = row[i];
+            if (colValue != null) {
+                row_hashcodes[i].set(hc.putString(colValue).hash().asBytes());
+            } else {
+                row_hashcodes[i].set(hc.putInt(0).hash().asBytes());
+            }
+        }
+
+        // use the row key column hash to get a consolidated hash for each cuboid
+        for (int i = 0; i < cuboidIds.length; i++) {
+            Hasher hc = hf.newHasher();
+            for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
+                hc.putBytes(row_hashcodes[allCuboidsBitSet[i][position]].array());
+            }
+
+            allCuboidsHLL[i].add(hc.hash().asBytes());
+        }
+    }
+
+    @Override
+    protected void doCleanup(Context context) throws IOException, InterruptedException {
+        ByteBuffer hllBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
+        HLLCounter hll;
+        for (int i = 0; i < cuboidIds.length; i++) {
+            hll = allCuboidsHLL[i];
+
+            outputKey.set(Bytes.toBytes(cuboidIds[i]));
+            hllBuf.clear();
+            hll.writeRegisters(hllBuf);
+            outputValue.set(hllBuf.array(), 0, hllBuf.position());
+            context.write(outputKey, outputValue);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidReducer.java
new file mode 100644
index 0000000..756c233
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidReducer.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.engine.mr.KylinReducer;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CubeStatsWriter;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class CalculateStatsFromBaseCuboidReducer extends KylinReducer<Text, Text, NullWritable, Text> {
+
+    private static final Logger logger = LoggerFactory.getLogger(CalculateStatsFromBaseCuboidReducer.class);
+
+    private KylinConfig cubeConfig;
+    protected long baseCuboidId;
+    protected Map<Long, HLLCounter> cuboidHLLMap = null;
+    private List<Long> baseCuboidRowCountInMappers;
+    private long totalRowsBeforeMerge = 0;
+
+    private String output = null;
+    private int samplingPercentage;
+
+    @Override
+    protected void setup(Context context) throws IOException {
+        super.bindCurrentConfiguration(context.getConfiguration());
+
+        Configuration conf = context.getConfiguration();
+        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+        String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+        CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
+        cubeConfig = cube.getConfig();
+
+        baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId();
+        baseCuboidRowCountInMappers = Lists.newLinkedList();
+
+        output = conf.get(BatchConstants.CFG_OUTPUT_PATH);
+        samplingPercentage = Integer
+                .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
+
+        cuboidHLLMap = Maps.newHashMap();
+    }
+
+    @Override
+    public void doReduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+        long cuboidId = Bytes.toLong(key.getBytes());
+        logger.info("Cuboid id to be processed: " + cuboidId);
+        for (Text value : values) {
+            HLLCounter hll = new HLLCounter(cubeConfig.getCubeStatsHLLPrecision());
+            ByteBuffer bf = ByteBuffer.wrap(value.getBytes(), 0, value.getLength());
+            hll.readRegisters(bf);
+
+            if (cuboidId == baseCuboidId) {
+                baseCuboidRowCountInMappers.add(hll.getCountEstimate());
+            }
+
+            totalRowsBeforeMerge += hll.getCountEstimate();
+
+            if (cuboidHLLMap.get(cuboidId) != null) {
+                cuboidHLLMap.get(cuboidId).merge(hll);
+            } else {
+                cuboidHLLMap.put(cuboidId, hll);
+            }
+        }
+    }
+
+    @Override
+    protected void doCleanup(Context context) throws IOException, InterruptedException {
+        long grandTotal = 0;
+        for (HLLCounter hll : cuboidHLLMap.values()) {
+            grandTotal += hll.getCountEstimate();
+        }
+        double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
+
+        CubeStatsWriter.writeCuboidStatistics(context.getConfiguration(), new Path(output), //
+                cuboidHLLMap, samplingPercentage, baseCuboidRowCountInMappers.size(), mapperOverlapRatio);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CopyDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CopyDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CopyDictionaryStep.java
new file mode 100644
index 0000000..78377ae
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CopyDictionaryStep.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+public class CopyDictionaryStep extends AbstractExecutable {
+
+    private static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
+
+    public CopyDictionaryStep() {
+        super();
+    }
+
+    @Override
+    protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+        final CubeManager mgr = CubeManager.getInstance(context.getConfig());
+        final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+        final CubeSegment optimizeSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+
+        CubeSegment oldSegment = optimizeSegment.getCubeInstance().getOriginalSegmentToOptimize(optimizeSegment);
+        Preconditions.checkNotNull(oldSegment,
+                "cannot find the original segment to be optimized by " + optimizeSegment);
+
+        // --- Copy dictionary
+        optimizeSegment.getDictionaries().putAll(oldSegment.getDictionaries());
+        optimizeSegment.getSnapshots().putAll(oldSegment.getSnapshots());
+
+        try {
+            CubeUpdate cubeBuilder = new CubeUpdate(cube);
+            cubeBuilder.setToUpdateSegs(optimizeSegment);
+            mgr.updateCube(cubeBuilder);
+        } catch (IOException e) {
+            logger.error("fail to merge dictionary or lookup snapshots", e);
+            return new ExecuteResult(e, e.getLocalizedMessage());
+        }
+
+        return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
index 65c5869..e06077a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
@@ -38,6 +38,7 @@ import javax.annotation.Nullable;
 public class CubingExecutableUtil {
 
     public static final String CUBE_NAME = "cubeName";
+    public static final String SEGMENT_NAME = "segmentName";
     public static final String SEGMENT_ID = "segmentId";
     public static final String MERGING_SEGMENT_IDS = "mergingSegmentIds";
     public static final String STATISTICS_PATH = "statisticsPath";
@@ -61,6 +62,14 @@ public class CubingExecutableUtil {
         return params.get(CUBE_NAME);
     }
 
+    public static void setSegmentName(String segmentName, Map<String, String> params) {
+        params.put(SEGMENT_NAME, segmentName);
+    }
+
+    public static String getSegmentName(Map<String, String> params) {
+        return params.get(SEGMENT_NAME);
+    }
+
     public static void setSegmentId(String segmentId, Map<String, String> params) {
         params.put(SEGMENT_ID, segmentId);
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 6a8ba4c..b49b639 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -28,12 +28,15 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.engine.mr.CubingJob;
 import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
 import org.apache.kylin.engine.mr.IMROutput2;
 import org.apache.kylin.engine.mr.MRUtil;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CuboidSchedulerUtil;
 import org.apache.kylin.job.execution.ExecutableManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -50,18 +53,27 @@ public class CuboidJob extends AbstractHadoopJob {
 
     private boolean skipped = false;
 
+    private CuboidScheduler cuboidScheduler;
+
     @Override
     public boolean isSkipped() {
         return skipped;
     }
 
-    private boolean checkSkip(String cubingJobId) {
+    private boolean checkSkip(String cubingJobId, int level) {
         if (cubingJobId == null)
             return false;
 
         ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
         CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId);
         skipped = cubingJob.isLayerCubing() == false;
+        if (!skipped) {
+            skipped = (level > cuboidScheduler.getBuildLevel());
+            if (skipped) {
+                logger.info("Job level: " + level + " for " + cubingJobId + "[" + cubingJobId
+                        + "] exceeds real cuboid tree levels : " + cuboidScheduler.getBuildLevel());
+            }
+        }
         return skipped;
     }
 
@@ -80,6 +92,7 @@ public class CuboidJob extends AbstractHadoopJob {
             options.addOption(OPTION_OUTPUT_PATH);
             options.addOption(OPTION_NCUBOID_LEVEL);
             options.addOption(OPTION_CUBING_JOB_ID);
+            options.addOption(OPTION_CUBOID_MODE);
             parseOptions(options, args);
 
             String output = getOptionValue(OPTION_OUTPUT_PATH);
@@ -87,12 +100,18 @@ public class CuboidJob extends AbstractHadoopJob {
             int nCuboidLevel = Integer.parseInt(getOptionValue(OPTION_NCUBOID_LEVEL));
             String segmentID = getOptionValue(OPTION_SEGMENT_ID);
             String cubingJobId = getOptionValue(OPTION_CUBING_JOB_ID);
+            String cuboidModeName = getOptionValue(OPTION_CUBOID_MODE);
+            if (cuboidModeName == null) {
+                cuboidModeName = CuboidModeEnum.CURRENT.toString();
+            }
 
             CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
             CubeInstance cube = cubeMgr.getCube(cubeName);
             CubeSegment segment = cube.getSegmentById(segmentID);
 
-            if (checkSkip(cubingJobId)) {
+            cuboidScheduler = CuboidSchedulerUtil.getCuboidSchedulerByMode(segment, cuboidModeName);
+
+            if (checkSkip(cubingJobId, nCuboidLevel)) {
                 logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + segmentID + "[" + segmentID + "]");
                 return 0;
             }
@@ -104,7 +123,7 @@ public class CuboidJob extends AbstractHadoopJob {
             setJobClasspath(job, cube.getConfig());
 
             // add metadata to distributed cache
-            attachSegmentMetadataWithDict(segment, job.getConfiguration());
+            attachSegmentMetadataWithAll(segment, job.getConfiguration());
 
             // Mapper
             job.setMapperClass(this.mapperClass);
@@ -122,12 +141,13 @@ public class CuboidJob extends AbstractHadoopJob {
 
             // set output
             IMROutput2.IMROutputFormat outputFormat = MRUtil.getBatchCubingOutputSide2(segment).getOuputFormat();
-            outputFormat.configureJobOutput(job, output, segment, nCuboidLevel);
+            outputFormat.configureJobOutput(job, output, segment, cuboidScheduler, nCuboidLevel);
 
             // set job configuration
             job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
             job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
             job.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, nCuboidLevel);
+            job.getConfiguration().set(BatchConstants.CFG_CUBOID_MODE, cuboidModeName);
 
             return waitForCompletion(job);
         } finally {

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 4e4c332..b5dc853 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -21,14 +21,14 @@ package org.apache.kylin.engine.mr.steps;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Collection;
-import java.util.List;
+import java.util.Set;
 
 import org.apache.hadoop.io.Text;
 import org.apache.kylin.common.KylinVersion;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.CuboidUtil;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.measure.BufferedMeasureCodec;
 import org.apache.kylin.measure.hllc.HLLCounter;
@@ -38,7 +38,6 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Lists;
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
@@ -57,7 +56,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
 
 
     protected boolean collectStatistics = false;
-    protected CuboidScheduler cuboidScheduler = null;
     protected int nRowKey;
     private Integer[][] allCuboidsBitSet = null;
     private HLLCounter[] allCuboidsHLL = null;
@@ -88,15 +86,11 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
         collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
         if (collectStatistics) {
             samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
-            cuboidScheduler = cubeSeg.getCuboidScheduler();
             nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
 
-            List<Long> cuboidIdList = Lists.newArrayList();
-            List<Integer[]> allCuboidsBitSetList = Lists.newArrayList();
-            addCuboidBitSet(baseCuboidId, allCuboidsBitSetList, cuboidIdList);
-
-            allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[cuboidIdList.size()][]);
-            cuboidIds = cuboidIdList.toArray(new Long[cuboidIdList.size()]);
+            Set<Long> cuboidIdSet = cubeSeg.getCuboidScheduler().getAllCuboidIds();
+            cuboidIds = cuboidIdSet.toArray(new Long[cuboidIdSet.size()]);
+            allCuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, nRowKey);
 
             allCuboidsHLL = new HLLCounter[cuboidIds.length];
             for (int i = 0; i < cuboidIds.length; i++) {
@@ -135,27 +129,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
 
     }
 
-    private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet, List<Long> allCuboids) {
-        allCuboids.add(cuboidId);
-        Integer[] indice = new Integer[Long.bitCount(cuboidId)];
-
-        long mask = Long.highestOneBit(baseCuboidId);
-        int position = 0;
-        for (int i = 0; i < nRowKey; i++) {
-            if ((mask & cuboidId) > 0) {
-                indice[position] = i;
-                position++;
-            }
-            mask = mask >> 1;
-        }
-
-        allCuboidsBitSet.add(indice);
-        Collection<Long> children = cuboidScheduler.getSpanningCuboid(cuboidId);
-        for (Long childId : children) {
-            addCuboidBitSet(childId, allCuboidsBitSet, allCuboids);
-        }
-    }
-
     @Override
     public void doMap(KEYIN key, Object record, Context context) throws IOException, InterruptedException {
         Collection<String[]> rowCollection = flatTableInputFormat.parseMapperInput(record);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 7f01c3a..f018afe 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -26,7 +26,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.base.Preconditions;
 import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.BytesWritable;
@@ -52,6 +51,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
@@ -110,6 +110,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
         if (collectStatistics && (taskId == numberOfTasks - 1)) {
             // hll
             isStatistics = true;
+            baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId();
             baseCuboidRowCountInMappers = Lists.newArrayList();
             cuboidHLLMap = Maps.newHashMap();
             samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
new file mode 100644
index 0000000..97f9dc1
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FilterRecommendCuboidDataJob extends AbstractHadoopJob {
+
+    private static final Logger logger = LoggerFactory.getLogger(FilterRecommendCuboidDataJob.class);
+
+    @Override
+    public int run(String[] args) throws Exception {
+        Options options = new Options();
+        try {
+            options.addOption(OPTION_JOB_NAME);
+            options.addOption(OPTION_CUBE_NAME);
+            options.addOption(OPTION_SEGMENT_ID);
+            options.addOption(OPTION_INPUT_PATH);
+            options.addOption(OPTION_OUTPUT_PATH);
+            parseOptions(options, args);
+
+            job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+            String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+            String segmentID = getOptionValue(OPTION_SEGMENT_ID);
+            Path input = new Path(getOptionValue(OPTION_INPUT_PATH));
+            Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+
+            CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+            CubeInstance cube = cubeMgr.getCube(cubeName);
+            CubeSegment optSegment = cube.getSegmentById(segmentID);
+            CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(optSegment);
+
+            logger.info("Starting: " + job.getJobName());
+
+            setJobClasspath(job, cube.getConfig());
+
+            // Mapper
+            job.setMapperClass(FilterRecommendCuboidDataMapper.class);
+
+            // Reducer
+            job.setNumReduceTasks(0);
+
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(Text.class);
+
+            // Input
+            job.setInputFormatClass(SequenceFileInputFormat.class);
+            FileInputFormat.setInputPaths(job, input);
+            // Output
+            job.setOutputFormatClass(SequenceFileOutputFormat.class);
+            FileOutputFormat.setOutputPath(job, output);
+
+            // set job configuration
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
+            // add metadata to distributed cache
+            attachSegmentMetadataWithDict(originalSegment, job.getConfiguration());
+
+            this.deletePath(job.getConfiguration(), output);
+
+            return waitForCompletion(job);
+        } catch (Exception e) {
+            logger.error("error in CuboidJob", e);
+            printUsage(options);
+            throw e;
+        } finally {
+            if (job != null)
+                cleanupTempConfFile(job.getConfiguration());
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
new file mode 100644
index 0000000..4aac9eb
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import static org.apache.kylin.engine.mr.JobBuilderSupport.PathNameCuboidBase;
+import static org.apache.kylin.engine.mr.JobBuilderSupport.PathNameCuboidOld;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+
+import com.google.common.base.Preconditions;
+
+public class FilterRecommendCuboidDataMapper extends KylinMapper<Text, Text, Text, Text> {
+
+    private MultipleOutputs mos;
+
+    private RowKeySplitter rowKeySplitter;
+    private long baseCuboid;
+    private Set<Long> recommendCuboids;
+
+    @Override
+    protected void setup(Context context) throws IOException {
+        super.bindCurrentConfiguration(context.getConfiguration());
+        mos = new MultipleOutputs(context);
+
+        String cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
+        String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
+
+        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+
+        CubeManager cubeManager = CubeManager.getInstance(config);
+        CubeInstance cube = cubeManager.getCube(cubeName);
+        CubeSegment optSegment = cube.getSegmentById(segmentID);
+        CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(optSegment);
+
+        rowKeySplitter = new RowKeySplitter(originalSegment, 65, 255);
+        baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
+
+        recommendCuboids = cube.getCuboidsRecommend();
+        Preconditions.checkNotNull(recommendCuboids, "The recommend cuboid map could not be null");
+    }
+
+    @Override
+    public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
+        long cuboidID = rowKeySplitter.split(key.getBytes());
+        if (cuboidID != baseCuboid && !recommendCuboids.contains(cuboidID)) {
+            return;
+        }
+
+        String baseOutputPath = PathNameCuboidOld;
+        if (cuboidID == baseCuboid) {
+            baseOutputPath = PathNameCuboidBase;
+        }
+        mos.write(key, value, generateFileName(baseOutputPath));
+    }
+
+    @Override
+    public void doCleanup(Context context) throws IOException, InterruptedException {
+        mos.close();
+
+        Path outputDirBase = new Path(context.getConfiguration().get(FileOutputFormat.OUTDIR), PathNameCuboidBase);
+        FileSystem fs = FileSystem.get(context.getConfiguration());
+        if (!fs.exists(outputDirBase)) {
+            fs.mkdirs(outputDirBase);
+            SequenceFile
+                    .createWriter(context.getConfiguration(),
+                            SequenceFile.Writer.file(new Path(outputDirBase, "part-m-00000")),
+                            SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class))
+                    .close();
+        }
+    }
+
+    private String generateFileName(String subDir) {
+        return subDir + "/part";
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidJob.java
new file mode 100644
index 0000000..62109f4
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidJob.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.engine.mr.ByteArrayWritable;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CuboidSchedulerUtil;
+import org.apache.kylin.engine.mr.common.MapReduceUtil;
+import org.apache.kylin.job.execution.ExecutableManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class InMemCuboidFromBaseCuboidJob extends AbstractHadoopJob {
+    protected static final Logger logger = LoggerFactory.getLogger(InMemCuboidFromBaseCuboidJob.class);
+
+    private boolean skipped = false;
+
+    @Override
+    public boolean isSkipped() {
+        return skipped;
+    }
+
+    private boolean checkSkip(String cubingJobId) {
+        if (cubingJobId == null)
+            return false;
+
+        ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
+        CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId);
+        skipped = !cubingJob.isInMemCubing();
+        return skipped;
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+        Options options = new Options();
+
+        try {
+            options.addOption(OPTION_JOB_NAME);
+            options.addOption(OPTION_CUBE_NAME);
+            options.addOption(OPTION_SEGMENT_ID);
+            options.addOption(OPTION_OUTPUT_PATH);
+            options.addOption(OPTION_CUBING_JOB_ID);
+            options.addOption(OPTION_INPUT_PATH);
+            options.addOption(OPTION_CUBOID_MODE);
+            parseOptions(options, args);
+
+            String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+            String segmentID = getOptionValue(OPTION_SEGMENT_ID);
+            String output = getOptionValue(OPTION_OUTPUT_PATH);
+
+            CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+            CubeInstance cube = cubeMgr.getCube(cubeName);
+            CubeSegment cubeSeg = cube.getSegmentById(segmentID);
+            String cubingJobId = getOptionValue(OPTION_CUBING_JOB_ID);
+
+            String cuboidModeName = getOptionValue(OPTION_CUBOID_MODE);
+            if (cuboidModeName == null) {
+                cuboidModeName = CuboidModeEnum.CURRENT.toString();
+            }
+
+            CuboidScheduler cuboidScheduler = CuboidSchedulerUtil.getCuboidSchedulerByMode(cubeSeg, cuboidModeName);
+
+            if (checkSkip(cubingJobId)) {
+                logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeSeg);
+                return 0;
+            }
+
+            job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+            logger.info("Starting: " + job.getJobName());
+
+            setJobClasspath(job, cube.getConfig());
+
+            // add metadata to distributed cache
+            attachSegmentMetadataWithAll(cubeSeg, job.getConfiguration());
+
+            // set job configuration
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+            job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
+            job.getConfiguration().set(BatchConstants.CFG_CUBOID_MODE, cuboidModeName);
+
+            String input = getOptionValue(OPTION_INPUT_PATH);
+            FileInputFormat.setInputPaths(job, new Path(input));
+            job.setInputFormatClass(SequenceFileInputFormat.class);
+
+            // set mapper
+            job.setMapperClass(InMemCuboidFromBaseCuboidMapper.class);
+            job.setMapOutputKeyClass(ByteArrayWritable.class);
+            job.setMapOutputValueClass(ByteArrayWritable.class);
+
+            // set output
+            job.setReducerClass(InMemCuboidFromBaseCuboidReducer.class);
+            job.setNumReduceTasks(MapReduceUtil.getInmemCubingReduceTaskNum(cubeSeg, cuboidScheduler));
+
+            // the cuboid file and KV class must be compatible with 0.7 version for smooth upgrade
+            job.setOutputFormatClass(SequenceFileOutputFormat.class);
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(Text.class);
+
+            Path outputPath = new Path(output);
+            FileOutputFormat.setOutputPath(job, outputPath);
+
+            HadoopUtil.deletePath(job.getConfiguration(), outputPath);
+
+            return waitForCompletion(job);
+        } catch (Exception e) {
+            logger.error("error in CuboidJob", e);
+            printUsage(options);
+            throw e;
+        } finally {
+            if (job != null)
+                cleanupTempConfFile(job.getConfiguration());
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        InMemCuboidFromBaseCuboidJob job = new InMemCuboidFromBaseCuboidJob();
+        int exitCode = ToolRunner.run(job, args);
+        System.exit(exitCode);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidMapper.java
new file mode 100644
index 0000000..05cee58
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidMapper.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.gridtable.CubeGridTable;
+import org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder;
+import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
+import org.apache.kylin.cube.inmemcubing.InputConverterUnit;
+import org.apache.kylin.cube.inmemcubing.InputConverterUnitForBaseCuboid;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
+import org.apache.kylin.engine.mr.ByteArrayWritable;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+public class InMemCuboidFromBaseCuboidMapper
+        extends InMemCuboidMapperBase<Text, Text, ByteArrayWritable, ByteArrayWritable, ByteArray> {
+    private static final Log logger = LogFactory.getLog(InMemCuboidFromBaseCuboidMapper.class);
+
+    private ByteBuffer keyValueBuffer;
+    private int keyOffset;
+
+    @Override
+    protected void setup(Mapper.Context context) throws IOException {
+        super.setup(context);
+
+        long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
+        GTInfo gtInfo = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, baseCuboid),
+                new CubeDimEncMap(cubeDesc, dictionaryMap));
+        keyValueBuffer = ByteBuffer.allocate(gtInfo.getMaxRecordLength());
+        keyOffset = cubeSegment.getRowKeyPreambleSize();
+    }
+
+    @Override
+    protected InputConverterUnit<ByteArray> getInputConverterUnit() {
+        return new InputConverterUnitForBaseCuboid();
+    }
+
+    @Override
+    protected Future getCubingThreadFuture(Context context, Map<TblColRef, Dictionary<String>> dictionaryMap,
+            int reserveMemoryMB, CuboidScheduler cuboidScheduler, InputConverterUnit<ByteArray> inputConverterUnit) {
+        AbstractInMemCubeBuilder cubeBuilder = new DoggedCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap);
+        cubeBuilder.setReserveMemoryMB(reserveMemoryMB);
+
+        ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true)
+                .setNameFormat("inmemory-cube-building-from-base-cuboid-mapper-%d").build());
+        return executorService.submit(cubeBuilder.buildAsRunnable(queue, inputConverterUnit,
+                new MapContextGTRecordWriter(context, cubeDesc, cubeSegment)));
+    }
+
+    @Override
+    protected ByteArray getRecordFromKeyValue(Text key, Text value) {
+        keyValueBuffer.clear();
+        keyValueBuffer.put(key.getBytes(), keyOffset, key.getBytes().length - keyOffset);
+        keyValueBuffer.put(value.getBytes());
+
+        byte[] keyValue = new byte[keyValueBuffer.position()];
+        System.arraycopy(keyValueBuffer.array(), 0, keyValue, 0, keyValueBuffer.position());
+
+        return new ByteArray(keyValue);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidReducer.java
new file mode 100644
index 0000000..fbc45d9
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidReducer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.mr.steps;
+
+public class InMemCuboidFromBaseCuboidReducer extends InMemCuboidReducer {
+    //pass
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index 73a2eb9..b0ea7b7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -93,7 +93,7 @@ public class InMemCuboidJob extends AbstractHadoopJob {
             setJobClasspath(job, cube.getConfig());
 
             // add metadata to distributed cache
-            attachSegmentMetadataWithDict(segment, job.getConfiguration());
+            attachSegmentMetadataWithAll(segment, job.getConfiguration());
 
             // set job configuration
             job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
@@ -116,7 +116,7 @@ public class InMemCuboidJob extends AbstractHadoopJob {
 
             // set output
             IMROutput2.IMROutputFormat outputFormat = MRUtil.getBatchCubingOutputSide2(segment).getOuputFormat();
-            outputFormat.configureJobOutput(job, output, segment, 0);
+            outputFormat.configureJobOutput(job, output, segment, segment.getCuboidScheduler(), 0);
 
             return waitForCompletion(job);
         } finally {

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
index 0642552..3dc95fa 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
@@ -33,8 +33,6 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
-import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
-import org.apache.kylin.cube.cuboid.TreeCuboidSchedulerManager;
 import org.apache.kylin.cube.inmemcubing.ConsumeBlockingQueueController;
 import org.apache.kylin.cube.inmemcubing.InputConverterUnit;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -43,7 +41,7 @@ import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.engine.mr.KylinMapper;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
+import org.apache.kylin.engine.mr.common.CuboidSchedulerUtil;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
@@ -113,11 +111,7 @@ public abstract class InMemCuboidMapperBase<KEYIN, VALUEIN, KEYOUT, VALUEOUT, T>
         }
 
         String cuboidModeName = conf.get(BatchConstants.CFG_CUBOID_MODE);
-        CuboidScheduler cuboidScheduler = TreeCuboidSchedulerManager.getTreeCuboidScheduler(cubeDesc, //
-                CuboidStatsReaderUtil.readCuboidStatsFromSegment(cube.getCuboidsByMode(cuboidModeName), cubeSegment));
-        if (cuboidScheduler == null) {
-            cuboidScheduler = new DefaultCuboidScheduler(cubeDesc);
-        }
+        CuboidScheduler cuboidScheduler = CuboidSchedulerUtil.getCuboidSchedulerByMode(cubeSegment, cuboidModeName);
 
         reserveMemoryMB = calculateReserveMB(conf);
         inputConverterUnit = getInputConverterUnit();

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
index 2058bc9..60d0870 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
@@ -95,7 +95,7 @@ public abstract class KVGTRecordWriter implements ICuboidWriter {
     protected abstract void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException;
 
     private void initVariables(Long cuboidId) {
-        rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeSegment, cuboidId));
+        rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findForMandatory(cubeDesc, cuboidId));
         keyBuf = rowKeyEncoder.createBuf();
 
         dimensions = Long.bitCount(cuboidId);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index c80283e..ef3adad 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -94,7 +94,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
     protected void setup(Context context) throws IOException, InterruptedException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
-        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
+        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
         segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
 
         config = AbstractHadoopJob.loadKylinPropsAndMetadata();

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
new file mode 100644
index 0000000..cf6e249
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.engine.mr.common.CubeStatsWriter;
+import org.apache.kylin.engine.mr.common.StatisticsDecisionUtil;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class MergeStatisticsWithOldStep extends AbstractExecutable {
+
+    private static final Logger logger = LoggerFactory.getLogger(MergeStatisticsWithOldStep.class);
+
+    protected Map<Long, HLLCounter> cuboidHLLMap = Maps.newHashMap();
+
+    public MergeStatisticsWithOldStep() {
+        super();
+    }
+
+    @Override
+    protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+        final CubeManager mgr = CubeManager.getInstance(context.getConfig());
+        final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+        final CubeSegment optimizeSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+        final String statsInputPath = CubingExecutableUtil.getStatisticsPath(this.getParams());
+
+        CubeSegment oldSegment = optimizeSegment.getCubeInstance().getOriginalSegmentToOptimize(optimizeSegment);
+        Preconditions.checkNotNull(oldSegment,
+                "cannot find the original segment to be optimized by " + optimizeSegment);
+
+        KylinConfig kylinConf = cube.getConfig();
+        Configuration conf = HadoopUtil.getCurrentConfiguration();
+        ResourceStore rs = ResourceStore.getStore(kylinConf);
+        int averageSamplingPercentage = 0;
+
+        try {
+            //1. Add statistics from optimized segment
+            Path statisticsFilePath = new Path(statsInputPath,
+                    BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
+            FileSystem hdfs = FileSystem.get(conf);
+            if (!hdfs.exists(statisticsFilePath))
+                throw new IOException("File " + statisticsFilePath + " does not exists");
+
+            CubeStatsReader optimizeSegmentStatsReader = new CubeStatsReader(optimizeSegment, null,
+                    optimizeSegment.getConfig(), statisticsFilePath);
+            averageSamplingPercentage += optimizeSegmentStatsReader.getSamplingPercentage();
+            addFromCubeStatsReader(optimizeSegmentStatsReader);
+
+            //2. Add statistics from old segment
+            CubeStatsReader oldSegmentStatsReader = new CubeStatsReader(oldSegment, null, oldSegment.getConfig());
+            averageSamplingPercentage += oldSegmentStatsReader.getSamplingPercentage();
+            addFromCubeStatsReader(oldSegmentStatsReader);
+
+            logger.info("Cuboid set with stats info: " + cuboidHLLMap.keySet().toString());
+            //3. Store merged statistics for recommend cuboids
+            averageSamplingPercentage = averageSamplingPercentage / 2;
+            Set<Long> cuboidsRecommend = cube.getCuboidsRecommend();
+
+            Map<Long, HLLCounter> resultCuboidHLLMap = Maps.newHashMapWithExpectedSize(cuboidsRecommend.size());
+            for (Long cuboid : cuboidsRecommend) {
+                HLLCounter hll = cuboidHLLMap.get(cuboid);
+                if (hll == null) {
+                    logger.warn("Cannot get the row count stats for cuboid " + cuboid);
+                } else {
+                    resultCuboidHLLMap.put(cuboid, hll);
+                }
+            }
+
+            String resultDir = CubingExecutableUtil.getMergedStatisticsPath(this.getParams());
+            CubeStatsWriter.writeCuboidStatistics(conf, new Path(resultDir), resultCuboidHLLMap,
+                    averageSamplingPercentage);
+
+            try (FSDataInputStream mergedStats = hdfs
+                    .open(new Path(resultDir, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME))) {
+                // put the statistics to metadata store
+                String statisticsFileName = optimizeSegment.getStatisticsResourcePath();
+                rs.putResource(statisticsFileName, mergedStats, System.currentTimeMillis());
+            }
+
+            //By default, the cube optimization will use in-memory cubing
+            CubingJob cubingJob = (CubingJob) getManager()
+                    .getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
+            StatisticsDecisionUtil.decideCubingAlgorithm(cubingJob, optimizeSegment);
+
+            return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+        } catch (IOException e) {
+            logger.error("fail to merge cuboid statistics", e);
+            return new ExecuteResult(e, e.getLocalizedMessage());
+        }
+
+    }
+
+    private void addFromCubeStatsReader(CubeStatsReader cubeStatsReader) {
+        for (Map.Entry<Long, HLLCounter> entry : cubeStatsReader.getCuboidRowEstimatesHLLOrigin().entrySet()) {
+            if (cuboidHLLMap.get(entry.getKey()) != null) {
+                cuboidHLLMap.get(entry.getKey()).merge(entry.getValue());
+            } else {
+                cuboidHLLMap.put(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 6680fd7..646c74f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -35,6 +35,7 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.mr.KylinMapper;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CuboidSchedulerUtil;
 import org.apache.kylin.engine.mr.common.NDCuboidBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -50,8 +51,8 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
     private Text outputKey = new Text();
     private String cubeName;
     private String segmentID;
-    private CubeSegment cubeSegment;
     private CubeDesc cubeDesc;
+    private CubeSegment cubeSegment;
     private CuboidScheduler cuboidScheduler;
 
     private int handleCounter;
@@ -65,17 +66,18 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
     protected void setup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
-        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
+        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
         segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
+        String cuboidModeName = context.getConfiguration().get(BatchConstants.CFG_CUBOID_MODE);
 
         KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
 
         CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
-        cubeSegment = cube.getSegmentById(segmentID);
         cubeDesc = cube.getDescriptor();
+        cubeSegment = cube.getSegmentById(segmentID);
         ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
         // initialize CubiodScheduler
-        cuboidScheduler = cubeSegment.getCuboidScheduler();
+        cuboidScheduler = CuboidSchedulerUtil.getCuboidSchedulerByMode(cubeSegment, cuboidModeName);
         rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
     }
 
@@ -104,7 +106,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
         }
 
         for (Long child : myChildren) {
-            Cuboid childCuboid = Cuboid.findById(cuboidScheduler, child);
+            Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child);
             Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
             outputKey.set(result.getSecond().array(), 0, result.getFirst());
             context.write(outputKey, value);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ReducerNumSizing.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ReducerNumSizing.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ReducerNumSizing.java
deleted file mode 100644
index 5c0555a..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ReducerNumSizing.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- *     http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.mr.steps;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.mr.common.CubeStatsReader;
-import org.apache.kylin.job.exception.JobException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ReducerNumSizing {
-
-    private static final Logger logger = LoggerFactory.getLogger(ReducerNumSizing.class);
-
-    public static int getLayeredCubingReduceTaskNum(CubeSegment cubeSegment, double totalMapInputMB, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
-        CubeDesc cubeDesc = cubeSegment.getCubeDesc();
-        KylinConfig kylinConfig = cubeDesc.getConfig();
-
-        double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
-        double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
-        logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio + ", level " + level);
-
-        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, kylinConfig);
-
-        double parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst;
-
-        if (level == -1) {
-            //merge case
-            double estimatedSize = cubeStatsReader.estimateCubeSize();
-            adjustedCurrentLayerSizeEst = estimatedSize > totalMapInputMB ? totalMapInputMB : estimatedSize;
-            logger.debug("estimated size {}, input size {}, adjustedCurrentLayerSizeEst: {}", estimatedSize, totalMapInputMB, adjustedCurrentLayerSizeEst);
-        } else if (level == 0) {
-            //base cuboid case TODO: the estimation could be very WRONG because it has no correction
-            adjustedCurrentLayerSizeEst = cubeStatsReader.estimateLayerSize(0);
-            logger.debug("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
-        } else {
-            parentLayerSizeEst = cubeStatsReader.estimateLayerSize(level - 1);
-            currentLayerSizeEst = cubeStatsReader.estimateLayerSize(level);
-            adjustedCurrentLayerSizeEst = totalMapInputMB / parentLayerSizeEst * currentLayerSizeEst;
-            logger.debug("totalMapInputMB: {}, parentLayerSizeEst: {}, currentLayerSizeEst: {}, adjustedCurrentLayerSizeEst: {}", totalMapInputMB, parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst);
-        }
-
-        // number of reduce tasks
-        int numReduceTasks = (int) Math.round(adjustedCurrentLayerSizeEst / perReduceInputMB * reduceCountRatio + 0.99);
-
-        // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
-        if (cubeDesc.hasMemoryHungryMeasures()) {
-            logger.debug("Multiply reducer num by 4 to boost performance for memory hungry measures");
-            numReduceTasks = numReduceTasks * 4;
-        }
-
-        // at least 1 reducer by default
-        numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
-        // no more than 500 reducer by default
-        numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
-
-        return numReduceTasks;
-    }
-
-    public static int getInmemCubingReduceTaskNum(CubeSegment cubeSeg) throws IOException {
-        KylinConfig kylinConfig = cubeSeg.getConfig();
-
-        Map<Long, Double> cubeSizeMap = new CubeStatsReader(cubeSeg, kylinConfig).getCuboidSizeMap();
-        double totalSizeInM = 0;
-        for (Double cuboidSize : cubeSizeMap.values()) {
-            totalSizeInM += cuboidSize;
-        }
-
-        double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
-
-        // number of reduce tasks
-        int numReduceTasks = (int) Math.round(totalSizeInM / perReduceInputMB);
-
-        // at least 1 reducer by default
-        numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
-        // no more than 500 reducer by default
-        numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
-
-        logger.info("Having total map input MB " + Math.round(totalSizeInM));
-        logger.info("Having per reduce MB " + perReduceInputMB);
-        logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + numReduceTasks);
-        return numReduceTasks;
-    }
-}


[17/22] kylin git commit: APACHE-KYLIN-2733: Introduce optimize job for adjusting cuboid set

Posted by nj...@apache.org.
APACHE-KYLIN-2733: Introduce optimize job for adjusting cuboid set


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/faf08625
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/faf08625
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/faf08625

Branch: refs/heads/yaho-cube-planner
Commit: faf08625f2df84a50a70bb93b54cdffd4f40e556
Parents: 74f581e
Author: Zhong <nj...@apache.org>
Authored: Wed Aug 30 11:17:43 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:04 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/CubeInstance.java     |  13 +-
 .../java/org/apache/kylin/cube/CubeManager.java |  82 ++++++-
 .../java/org/apache/kylin/cube/CubeUpdate.java  |  14 +-
 .../kylin/cube/common/RowKeySplitter.java       |   2 +-
 .../apache/kylin/cube/cuboid/CuboidUtil.java    |  48 ++++
 .../cube/cuboid/TreeCuboidSchedulerManager.java |   6 +-
 .../cube/inmemcubing/InMemCubeBuilder.java      |   2 +-
 .../org/apache/kylin/cube/kv/RowKeyDecoder.java |   4 +-
 .../org/apache/kylin/engine/EngineFactory.java  |   4 +
 .../apache/kylin/engine/IBatchCubingEngine.java |   3 +
 .../kylin/job/constant/ExecutableConstants.java |   5 +
 .../org/apache/kylin/job/dao/ExecutableDao.java |  18 +-
 .../org/apache/kylin/job/dao/ExecutablePO.java  |  11 +
 .../kylin/job/execution/ExecutableManager.java  |  41 +++-
 .../job/impl/threadpool/DefaultScheduler.java   |   7 +-
 .../engine/mr/BatchOptimizeJobBuilder2.java     | 226 +++++++++++++++++
 .../mr/BatchOptimizeJobCheckpointBuilder.java   |  89 +++++++
 .../org/apache/kylin/engine/mr/CubingJob.java   |   7 +-
 .../org/apache/kylin/engine/mr/IMROutput2.java  |  13 +-
 .../kylin/engine/mr/JobBuilderSupport.java      |  51 +++-
 .../kylin/engine/mr/MRBatchCubingEngine.java    |   5 +
 .../kylin/engine/mr/MRBatchCubingEngine2.java   |   5 +
 .../java/org/apache/kylin/engine/mr/MRUtil.java |   4 +
 .../engine/mr/common/AbstractHadoopJob.java     |  17 ++
 .../kylin/engine/mr/common/BatchConstants.java  |   1 +
 .../kylin/engine/mr/common/CubeStatsReader.java | 126 ++++++----
 .../engine/mr/common/CuboidRecommenderUtil.java |  14 +-
 .../engine/mr/common/CuboidSchedulerUtil.java   |  54 ++++
 .../engine/mr/common/CuboidStatsReaderUtil.java |   5 +-
 .../kylin/engine/mr/common/MapReduceUtil.java   | 117 +++++++++
 .../mr/common/StatisticsDecisionUtil.java       |   2 +-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |   2 +-
 .../steps/CalculateStatsFromBaseCuboidJob.java  | 116 +++++++++
 .../CalculateStatsFromBaseCuboidMapper.java     | 166 +++++++++++++
 .../CalculateStatsFromBaseCuboidReducer.java    | 112 +++++++++
 .../engine/mr/steps/CopyDictionaryStep.java     |  69 ++++++
 .../engine/mr/steps/CubingExecutableUtil.java   |   9 +
 .../apache/kylin/engine/mr/steps/CuboidJob.java |  28 ++-
 .../mr/steps/FactDistinctColumnsMapper.java     |  37 +--
 .../mr/steps/FactDistinctColumnsReducer.java    |   3 +-
 .../mr/steps/FilterRecommendCuboidDataJob.java  | 103 ++++++++
 .../steps/FilterRecommendCuboidDataMapper.java  | 107 ++++++++
 .../mr/steps/InMemCuboidFromBaseCuboidJob.java  | 154 ++++++++++++
 .../steps/InMemCuboidFromBaseCuboidMapper.java  |  95 ++++++++
 .../steps/InMemCuboidFromBaseCuboidReducer.java |  23 ++
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |   4 +-
 .../engine/mr/steps/InMemCuboidMapperBase.java  |  10 +-
 .../kylin/engine/mr/steps/KVGTRecordWriter.java |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java      |   2 +-
 .../mr/steps/MergeStatisticsWithOldStep.java    | 144 +++++++++++
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |  12 +-
 .../kylin/engine/mr/steps/ReducerNumSizing.java | 106 --------
 .../UpdateCubeInfoAfterCheckpointStep.java      |  69 ++++++
 .../steps/UpdateCubeInfoAfterOptimizeStep.java  |  72 ++++++
 .../mr/steps/UpdateOldCuboidShardJob.java       | 105 ++++++++
 .../mr/steps/UpdateOldCuboidShardMapper.java    | 142 +++++++++++
 .../engine/spark/SparkBatchCubingEngine.java    |   6 +
 .../kylin/rest/controller/CubeController.java   |  60 +++++
 .../kylin/rest/request/JobOptimizeRequest.java  |  34 +++
 .../apache/kylin/rest/service/CubeService.java  |   4 +
 .../apache/kylin/rest/service/JobService.java   | 244 ++++++++++++++++++-
 .../storage/hbase/steps/CreateHTableJob.java    |  19 +-
 .../hbase/steps/HBaseMROutput2Transition.java   |  37 ++-
 .../kylin/storage/hbase/steps/HBaseMRSteps.java |  68 ++++++
 64 files changed, 2911 insertions(+), 249 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 05fb5be..e82144e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -146,6 +146,17 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         return segments.getMergingSegments(mergedSegment);
     }
 
+    public CubeSegment getOriginalSegmentToOptimize(CubeSegment optimizedSegment) {
+        for (CubeSegment segment : this.getSegments(SegmentStatusEnum.READY)) {
+            if (!optimizedSegment.equals(segment) //
+                    && optimizedSegment.getDateRangeStart() == segment.getDateRangeStart()
+                    && optimizedSegment.getDateRangeEnd() == segment.getDateRangeEnd()) {
+                return segment;
+            }
+        }
+        return null;
+    }
+
     public CubeDesc getDescriptor() {
         return CubeDescManager.getInstance(config).getCubeDesc(descName);
     }
@@ -351,7 +362,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
             return cuboidsRecommend;
         case RECOMMEND_MISSING_WITH_BASE:
             cuboidsRecommend.removeAll(currentCuboids);
-            currentCuboids.add(getCuboidScheduler().getBaseCuboidId());
+            cuboidsRecommend.add(getCuboidScheduler().getBaseCuboidId());
             return cuboidsRecommend;
         default:
             return null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a679dde..b815e75 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -75,6 +75,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
@@ -425,6 +426,10 @@ public class CubeManager implements IRealizationProvider {
             cube.setCuboids(update.getCuboids());
         }
 
+        if (update.getCuboidsRecommend() != null) {
+            cube.setCuboidsRecommend(update.getCuboidsRecommend());
+        }
+
         try {
             getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
         } catch (IllegalStateException ise) {
@@ -503,6 +508,29 @@ public class CubeManager implements IRealizationProvider {
         return newSegment;
     }
 
+    public CubeSegment[] optimizeSegments(CubeInstance cube, Set<Long> cuboidsRecommend) throws IOException {
+        checkReadyForOptimize(cube);
+
+        List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY);
+        CubeSegment[] optimizeSegments = new CubeSegment[readySegments.size()];
+        int i = 0;
+        for (CubeSegment segment : readySegments) {
+            CubeSegment newSegment = newSegment(cube, segment.getDateRangeStart(), segment.getDateRangeEnd(), 0, 0);
+            newSegment.setSourcePartitionOffsetStart(null);
+            newSegment.setSourcePartitionOffsetEnd(null);
+            validateNewSegments(cube, newSegment);
+
+            optimizeSegments[i++] = newSegment;
+        }
+
+        CubeUpdate cubeBuilder = new CubeUpdate(cube);
+        cubeBuilder.setCuboidsRecommend(cuboidsRecommend);
+        cubeBuilder.setToAddSegs(optimizeSegments);
+        updateCube(cubeBuilder);
+
+        return optimizeSegments;
+    }
+
     public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset)
             throws IOException {
         checkBuildingSegment(cube);
@@ -644,8 +672,15 @@ public class CubeManager implements IRealizationProvider {
     }
 
     private void checkBuildingSegment(CubeInstance cube) {
-        int maxBuldingSeg = cube.getConfig().getMaxBuildingSegments();
-        if (cube.getBuildingSegments().size() >= maxBuldingSeg) {
+        checkBuildingSegment(cube, cube.getConfig().getMaxBuildingSegments());
+    }
+
+    public void checkReadyForOptimize(CubeInstance cube) {
+        checkBuildingSegment(cube, 1);
+    }
+
+    private void checkBuildingSegment(CubeInstance cube, int maxBuildingSeg) {
+        if (cube.getBuildingSegments().size() >= maxBuildingSeg) {
             throw new IllegalStateException(
                     "There is already " + cube.getBuildingSegments().size() + " building segment; ");
         }
@@ -771,6 +806,49 @@ public class CubeManager implements IRealizationProvider {
         updateCube(cubeBuilder);
     }
 
+    public void promoteNewlyOptimizeSegments(CubeInstance cube, CubeSegment... optimizedSegments) throws IOException {
+        for (CubeSegment seg : optimizedSegments) {
+            seg.setStatus(SegmentStatusEnum.READY_PENDING);
+        }
+
+        CubeUpdate cubeBuilder = new CubeUpdate(cube);
+        cubeBuilder.setToUpdateSegs(optimizedSegments);
+        updateCube(cubeBuilder);
+    }
+
+    public void promoteCheckpointOptimizeSegments(CubeInstance cube, Map<Long, Long> recommendCuboids,
+            CubeSegment... optimizedSegments) throws IOException {
+        if (cube.getSegments().size() != optimizedSegments.length * 2) {
+            throw new IllegalStateException("For cube " + cube
+                    + ", every READY segment should be optimized and all segments should be READY before optimizing");
+        }
+        CubeSegment[] originalSegments = new CubeSegment[optimizedSegments.length];
+        int i = 0;
+        for (CubeSegment seg : optimizedSegments) {
+            originalSegments[i++] = cube.getOriginalSegmentToOptimize(seg);
+
+            if (StringUtils.isBlank(seg.getStorageLocationIdentifier()))
+                throw new IllegalStateException(
+                        "For cube " + cube + ", segment " + seg + " missing StorageLocationIdentifier");
+
+            if (StringUtils.isBlank(seg.getLastBuildJobID()))
+                throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing LastBuildJobID");
+
+            seg.setStatus(SegmentStatusEnum.READY);
+        }
+
+        logger.info("Promoting cube " + cube + ", new segments " + Arrays.toString(optimizedSegments)
+                + ", to remove segments " + originalSegments);
+
+        CubeUpdate cubeBuilder = new CubeUpdate(cube);
+        cubeBuilder.setToRemoveSegs(originalSegments) //
+                .setToUpdateSegs(optimizedSegments) //
+                .setStatus(RealizationStatusEnum.READY) //
+                .setCuboids(recommendCuboids) //
+                .setCuboidsRecommend(Sets.<Long> newHashSet());
+        updateCube(cubeBuilder);
+    }
+
     public void validateNewSegments(CubeInstance cube, CubeSegment newSegments) {
         List<CubeSegment> tobe = cube.calculateToBeSegments(newSegments);
         List<CubeSegment> newList = Arrays.asList(newSegments);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
index fae20dc..2e1d652 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.cube;
 
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 
@@ -34,6 +35,7 @@ public class CubeUpdate {
     private String owner;
     private int cost = -1;
     private Map<Long, Long> cuboids = null;
+    private Set<Long> cuboidsRecommend = null;
 
     public CubeUpdate(CubeInstance cubeInstance) {
         this.cubeInstance = cubeInstance;
@@ -106,7 +108,17 @@ public class CubeUpdate {
         return cuboids;
     }
 
-    public void setCuboids(Map<Long, Long> cuboids) {
+    public CubeUpdate setCuboids(Map<Long, Long> cuboids) {
         this.cuboids = cuboids;
+        return this;
+    }
+
+    public Set<Long> getCuboidsRecommend() {
+        return cuboidsRecommend;
+    }
+
+    public CubeUpdate setCuboidsRecommend(Set<Long> cuboidsRecommend) {
+        this.cuboidsRecommend = cuboidsRecommend;
+        return this;
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index cd26347..0c54ecf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -115,7 +115,7 @@ public class RowKeySplitter implements java.io.Serializable {
         offset += RowConstants.ROWKEY_CUBOIDID_LEN;
 
         long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
-        Cuboid cuboid = Cuboid.findById(cubeSegment, lastSplittedCuboidId);
+        Cuboid cuboid = Cuboid.findForMandatory(cubeDesc, lastSplittedCuboidId);
 
         // rowkey columns
         for (int i = 0; i < cuboid.getColumns().size(); i++) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidUtil.java
new file mode 100644
index 0000000..a84f153
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.cuboid;
+
+import com.google.common.base.Preconditions;
+
+public class CuboidUtil {
+
+    public static Integer[][] getCuboidBitSet(Long[] cuboidIds, int nRowKey) {
+        Preconditions.checkArgument(nRowKey < Long.SIZE,
+                "the size of row key could not be large than " + (Long.SIZE - 1));
+
+        Integer[][] allCuboidsBitSet = new Integer[cuboidIds.length][];
+
+        for (int j = 0; j < cuboidIds.length; j++) {
+            Long cuboidId = cuboidIds[j];
+
+            allCuboidsBitSet[j] = new Integer[Long.bitCount(cuboidId)];
+
+            long mask = 1L << (nRowKey - 1);
+            int position = 0;
+            for (int i = 0; i < nRowKey; i++) {
+                if ((mask & cuboidId) > 0) {
+                    allCuboidsBitSet[j][position] = i;
+                    position++;
+                }
+                mask = mask >> 1;
+            }
+        }
+        return allCuboidsBitSet;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
index 5e8d965..22e636b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
@@ -64,7 +64,7 @@ public class TreeCuboidSchedulerManager {
      * @param cubeName
      * @return null if the cube has no pre-built cuboids
      */
-    public static TreeCuboidScheduler getTreeCuboidScheduler(String cubeName) {
+    public TreeCuboidScheduler getTreeCuboidScheduler(String cubeName) {
         TreeCuboidScheduler result = cache.get(cubeName);
         if (result == null) {
             CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
@@ -83,14 +83,14 @@ public class TreeCuboidSchedulerManager {
         return result;
     }
 
-    public static TreeCuboidScheduler getTreeCuboidScheduler(CubeDesc cubeDesc, Map<Long, Long> cuboidsWithRowCnt) {
+    public TreeCuboidScheduler getTreeCuboidScheduler(CubeDesc cubeDesc, Map<Long, Long> cuboidsWithRowCnt) {
         if (cuboidsWithRowCnt == null || cuboidsWithRowCnt.isEmpty()) {
             return null;
         }
         return getTreeCuboidScheduler(cubeDesc, Lists.newArrayList(cuboidsWithRowCnt.keySet()), cuboidsWithRowCnt);
     }
 
-    public static TreeCuboidScheduler getTreeCuboidScheduler(CubeDesc cubeDesc, List<Long> cuboidIds,
+    public TreeCuboidScheduler getTreeCuboidScheduler(CubeDesc cubeDesc, List<Long> cuboidIds,
             Map<Long, Long> cuboidsWithRowCnt) {
         if (cuboidIds == null || cuboidsWithRowCnt == null) {
             return null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index f63b53f..97bb1de 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -108,7 +108,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
-        GTInfo info = CubeGridTable.newGTInfo(Cuboid.findById(cuboidScheduler, cuboidID),
+        GTInfo info = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, cuboidID),
                 new CubeDimEncMap(cubeDesc, dictionaryMap)
         );
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index 5a1f668..bb03c4c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -36,7 +36,6 @@ import org.apache.kylin.metadata.model.TblColRef;
  */
 public class RowKeyDecoder {
 
-    private final CubeSegment cubeSegment;
     private final CubeDesc cubeDesc;
     private final RowKeyColumnIO colIO;
     private final RowKeySplitter rowKeySplitter;
@@ -45,7 +44,6 @@ public class RowKeyDecoder {
     private List<String> values;
 
     public RowKeyDecoder(CubeSegment cubeSegment) {
-        this.cubeSegment = cubeSegment;
         this.cubeDesc = cubeSegment.getCubeDesc();
         this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
         this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
@@ -75,7 +73,7 @@ public class RowKeyDecoder {
         if (this.cuboid != null && this.cuboid.getId() == cuboidID) {
             return;
         }
-        this.cuboid = Cuboid.findById(cubeSegment, cuboidID);
+        this.cuboid = Cuboid.findForMandatory(cubeDesc, cuboidID);
     }
 
     private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
index 78b1efe..03d986b 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
@@ -61,4 +61,8 @@ public class EngineFactory {
         return batchEngine(mergeSegment).createBatchMergeJob(mergeSegment, submitter);
     }
 
+    /** Optimize a segment based on the cuboid recommend list produced by the cube planner. */
+    public static DefaultChainedExecutable createBatchOptimizeJob(CubeSegment optimizeSegment, String submitter) {
+        return batchEngine(optimizeSegment).createBatchOptimizeJob(optimizeSegment, submitter);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
index 754dbde..a618eac 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
@@ -37,6 +37,9 @@ public interface IBatchCubingEngine {
     /** Merge multiple small segments into a big one. */
     public DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter);
 
+    /** Optimize a segment based on the cuboid recommend list produced by the cube planner. */
+    public DefaultChainedExecutable createBatchOptimizeJob(CubeSegment optimizeSegment, String submitter);
+
     public Class<?> getSourceInterface();
 
     public Class<?> getStorageInterface();

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 2de3efa..b39e3aa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -36,6 +36,9 @@ public final class ExecutableConstants {
     public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table";
     public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables";
     public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns";
+    public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = "Calculate Statistics from Base Cuboid";
+    public static final String STEP_NAME_FILTER_RECOMMEND_CUBOID_DATA_FOR_OPTIMIZATION = "Filter Recommend Cuboid Data for Optimization";
+    public static final String STEP_NAME_UPDATE_OLD_CUBOID_SHARD = "Update Old Cuboid Shard for Optimization";
     public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid";
     public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube In-Mem";
     public static final String STEP_NAME_BUILD_SPARK_CUBE = "Build Cube with Spark";
@@ -44,8 +47,10 @@ public final class ExecutableConstants {
     public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
     public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert Cuboid Data to HFile";
     public static final String STEP_NAME_BULK_LOAD_HFILE = "Load HFile to HBase Table";
+    public static final String STEP_NAME_COPY_DICTIONARY = "Copy dictionary from Old Segment";
     public static final String STEP_NAME_MERGE_DICTIONARY = "Merge Cuboid Dictionary";
     public static final String STEP_NAME_MERGE_STATISTICS = "Merge Cuboid Statistics";
+    public static final String STEP_NAME_MERGE_STATISTICS_WITH_OLD = "Merge Cuboid Statistics with Old for Optimization";
     public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid Statistics";
     public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
     public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index bd020f8..60ccaaf 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -92,8 +92,8 @@ public class ExecutableDao {
         return store.getResource(path, ExecutablePO.class, JOB_SERIALIZER);
     }
 
-    private void writeJobResource(String path, ExecutablePO job) throws IOException {
-        store.putResource(path, job, JOB_SERIALIZER);
+    private long writeJobResource(String path, ExecutablePO job) throws IOException {
+        return store.putResource(path, job, JOB_SERIALIZER);
     }
 
     private ExecutableOutputPO readJobOutputResource(String path) throws IOException {
@@ -179,6 +179,20 @@ public class ExecutableDao {
         }
     }
 
+    public ExecutablePO updateJob(ExecutablePO job) throws PersistentException {
+        try {
+            if (getJob(job.getUuid()) == null) {
+                throw new IllegalArgumentException("job id:" + job.getUuid() + " does not exist");
+            }
+            final long ts = writeJobResource(pathOfJob(job), job);
+            job.setLastModified(ts);
+            return job;
+        } catch (IOException e) {
+            logger.error("error update job:" + job.getUuid(), e);
+            throw new PersistentException(e);
+        }
+    }
+
     public void deleteJob(String uuid) throws PersistentException {
         try {
             store.deleteResource(pathOfJob(uuid));

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
index 75717e0..f48c876 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
@@ -39,6 +39,9 @@ public class ExecutablePO extends RootPersistentEntity {
     @JsonProperty("tasks")
     private List<ExecutablePO> tasks;
 
+    @JsonProperty("tasks_check")
+    private List<ExecutablePO> tasksForCheck;
+
     @JsonProperty("type")
     private String type;
 
@@ -61,6 +64,14 @@ public class ExecutablePO extends RootPersistentEntity {
         this.tasks = tasks;
     }
 
+    public List<ExecutablePO> getTasksForCheck() {
+        return tasksForCheck;
+    }
+
+    public void setTasksForCheck(List<ExecutablePO> tasksForCheck) {
+        this.tasksForCheck = tasksForCheck;
+    }
+
     public String getType() {
         return type;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 124a5a9..bd2a2d3 100755
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -101,6 +101,13 @@ public class ExecutableManager {
             }
             result.setTasks(tasks);
         }
+        if (executable instanceof CheckpointExecutable) {
+            List<ExecutablePO> tasksForCheck = Lists.newArrayList();
+            for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) {
+                tasksForCheck.add(parse(taskForCheck));
+            }
+            result.setTasksForCheck(tasksForCheck);
+        }
         return result;
     }
 
@@ -126,6 +133,23 @@ public class ExecutableManager {
         }
     }
 
+    public void updateCheckpointJob(String jobId, List<AbstractExecutable> subTasksForCheck) {
+        try {
+            final ExecutablePO job = executableDao.getJob(jobId);
+            Preconditions.checkArgument(job != null, "there is no related job for job id:" + jobId);
+
+            List<ExecutablePO> tasksForCheck = Lists.newArrayListWithExpectedSize(subTasksForCheck.size());
+            for (AbstractExecutable taskForCheck : subTasksForCheck) {
+                tasksForCheck.add(parse(taskForCheck));
+            }
+            job.setTasksForCheck(tasksForCheck);
+            executableDao.updateJob(job);
+        } catch (PersistentException e) {
+            logger.error("fail to update checkpoint job:" + jobId, e);
+            throw new RuntimeException(e);
+        }
+    }
+
     //for ut
     public void deleteJob(String jobId) {
         try {
@@ -353,7 +377,15 @@ public class ExecutableManager {
         if (job == null) {
             return;
         }
-
+        if (job.getStatus().isFinalState()) {
+            if (job.getStatus() != ExecutableState.DISCARDED) {
+                logger.warn("The status of job " + jobId + " is " + job.getStatus().toString()
+                        + ". It's final state and cannot be transfer to be discarded!!!");
+            } else {
+                logger.warn("The job " + jobId + " has been discarded.");
+            }
+            return;
+        }
         if (job instanceof DefaultChainedExecutable) {
             List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks();
             for (AbstractExecutable task : tasks) {
@@ -505,6 +537,13 @@ public class ExecutableManager {
                     ((ChainedExecutable) result).addTask(parseTo(subTask));
                 }
             }
+            List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
+            if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
+                Preconditions.checkArgument(result instanceof CheckpointExecutable);
+                for (ExecutablePO subTaskForCheck : tasksForCheck) {
+                    ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
+                }
+            }
             return result;
         } catch (ReflectiveOperationException e) {
             throw new IllegalStateException("cannot parse this job:" + executablePO.getId(), e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index b119e1b..315671c 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -88,8 +88,9 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
                         nRunning++;
                         continue;
                     }
-                    final Output output = executableManager.getOutput(id);
-                    if ((output.getState() != ExecutableState.READY)) {
+                    final AbstractExecutable executable = executableManager.getJob(id);
+                    if (!executable.isReady()) {
+                        final Output output = executableManager.getOutput(id);
                         // logger.debug("Job id:" + id + " not runnable");
                         if (output.getState() == ExecutableState.DISCARDED) {
                             nDiscarded++;
@@ -105,10 +106,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
                         continue;
                     }
                     nReady++;
-                    AbstractExecutable executable = null;
                     String jobDesc = null;
                     try {
-                        executable = executableManager.getJob(id);
                         jobDesc = executable.toString();
                         logger.info(jobDesc + " prepare to schedule");
                         context.addRunningJob(executable);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobBuilder2.java
new file mode 100644
index 0000000..645525b
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobBuilder2.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.engine.mr.steps.CopyDictionaryStep;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.engine.mr.steps.FilterRecommendCuboidDataJob;
+import org.apache.kylin.engine.mr.steps.InMemCuboidFromBaseCuboidJob;
+import org.apache.kylin.engine.mr.steps.MergeStatisticsWithOldStep;
+import org.apache.kylin.engine.mr.steps.NDCuboidJob;
+import org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterOptimizeStep;
+import org.apache.kylin.engine.mr.steps.UpdateOldCuboidShardJob;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+public class BatchOptimizeJobBuilder2 extends JobBuilderSupport {
+    private static final Logger logger = LoggerFactory.getLogger(BatchOptimizeJobBuilder2.class);
+
+    private final IMROutput2.IMRBatchOptimizeOutputSide2 outputSide;
+
+    public BatchOptimizeJobBuilder2(CubeSegment optimizeSegment, String submitter) {
+        super(optimizeSegment, submitter);
+        this.outputSide = MRUtil.getBatchOptimizeOutputSide2(optimizeSegment);
+    }
+
+    public CubingJob build() {
+        logger.info("MR_V2 new job to OPTIMIZE a segment " + seg);
+
+        final CubingJob result = CubingJob.createOptimizeJob(seg, submitter, config);
+        CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+
+        final String jobId = result.getId();
+        final String cuboidRootPath = getCuboidRootPath(jobId);
+        final String optimizeCuboidRootPath = getOptimizationCuboidPath(jobId);
+
+        CubeSegment oldSegment = seg.getCubeInstance().getOriginalSegmentToOptimize(seg);
+        Preconditions.checkNotNull(oldSegment, "cannot find the original segment to be optimized by " + seg);
+
+        // Phase 1: Prepare base cuboid data from old segment
+        String oldcuboidRootPath = getCuboidRootPath(oldSegment) + "*";
+        result.addTask(createFilterRecommendCuboidDataStep(oldcuboidRootPath, optimizeCuboidRootPath));
+
+        // Phase 2: Prepare dictionary and statistics for new segment
+        result.addTask(createCopyDictionaryStep());
+        String optStatsSourcePath = getBaseCuboidPath(optimizeCuboidRootPath);
+        String optStatsDstPath = getOptimizationStatisticsPath(jobId);
+        result.addTask(createCalculateStatsFromBaseCuboid(optStatsSourcePath, optStatsDstPath,
+                CuboidModeEnum.RECOMMEND_MISSING));
+        result.addTask(createMergeStatisticsWithOldStep(jobId, optStatsDstPath, getStatisticsPath(jobId)));
+        outputSide.addStepPhase1_CreateHTable(result);
+
+        result.addTask(createUpdateShardForOldCuboidDataStep(optimizeCuboidRootPath + "*", cuboidRootPath));
+
+        // Phase 3: Build Cube for Missing Cuboid Data
+        addLayerCubingSteps(result, jobId, CuboidModeEnum.RECOMMEND_MISSING_WITH_BASE, cuboidRootPath); // layer cubing
+        result.addTask(createInMemCubingStep(jobId, CuboidModeEnum.RECOMMEND_MISSING_WITH_BASE, cuboidRootPath));
+
+        outputSide.addStepPhase2_BuildCube(result);
+
+        // Phase 4: Update Metadata & Cleanup
+        result.addTask(createUpdateCubeInfoAfterOptimizeStep(jobId));
+
+        return result;
+    }
+
+    public MapReduceExecutable createFilterRecommendCuboidDataStep(String inputPath, String outputPath) {
+        MapReduceExecutable result = new MapReduceExecutable();
+        result.setName(ExecutableConstants.STEP_NAME_FILTER_RECOMMEND_CUBOID_DATA_FOR_OPTIMIZATION);
+
+        StringBuilder cmd = new StringBuilder();
+        appendMapReduceParameters(cmd);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
+                "Kylin_Filter_Recommend_Cuboid_Data_" + seg.getRealization().getName());
+
+        result.setMapReduceParams(cmd.toString());
+        result.setMapReduceJobClass(FilterRecommendCuboidDataJob.class);
+        return result;
+    }
+
+    public CopyDictionaryStep createCopyDictionaryStep() {
+        CopyDictionaryStep result = new CopyDictionaryStep();
+        result.setName(ExecutableConstants.STEP_NAME_COPY_DICTIONARY);
+
+        CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+        CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+        return result;
+    }
+
+    private MapReduceExecutable createUpdateShardForOldCuboidDataStep(String inputPath, String outputPath) {
+        MapReduceExecutable result = new MapReduceExecutable();
+        result.setName(ExecutableConstants.STEP_NAME_UPDATE_OLD_CUBOID_SHARD);
+
+        StringBuilder cmd = new StringBuilder();
+        appendMapReduceParameters(cmd);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
+                "Kylin_Update_Old_Cuboid_Shard_" + seg.getRealization().getName());
+
+        result.setMapReduceParams(cmd.toString());
+        result.setMapReduceJobClass(UpdateOldCuboidShardJob.class);
+        return result;
+    }
+
+    private MergeStatisticsWithOldStep createMergeStatisticsWithOldStep(final String jobId, final String optStatsPath,
+            final String mergedStatisticsFolder) {
+        MergeStatisticsWithOldStep result = new MergeStatisticsWithOldStep();
+        result.setName(ExecutableConstants.STEP_NAME_MERGE_STATISTICS_WITH_OLD);
+
+        CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
+        CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+        CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+        CubingExecutableUtil.setStatisticsPath(optStatsPath, result.getParams());
+        CubingExecutableUtil.setMergedStatisticsPath(mergedStatisticsFolder, result.getParams());
+
+        return result;
+    }
+
+    private void addLayerCubingSteps(final CubingJob result, final String jobId, final CuboidModeEnum cuboidMode,
+            final String cuboidRootPath) {
+        // Don't know statistics so that tree cuboid scheduler is not determined. Determine the maxLevel at runtime
+        final int maxLevel = seg.getCubeDesc().getRowkey().getRowKeyColumns().length;
+        // Don't need to build base cuboid
+        // n dim cuboid steps
+        for (int i = 1; i <= maxLevel; i++) {
+            String parentCuboidPath = i == 1 ? getBaseCuboidPath(cuboidRootPath)
+                    : getCuboidOutputPathsByLevel(cuboidRootPath, i - 1);
+            result.addTask(createNDimensionCuboidStep(parentCuboidPath,
+                    getCuboidOutputPathsByLevel(cuboidRootPath, i), i, jobId, cuboidMode));
+        }
+    }
+
+    private MapReduceExecutable createNDimensionCuboidStep(String parentPath, String outputPath, int level,
+            String jobId, CuboidModeEnum cuboidMode) {
+        // ND cuboid job
+        MapReduceExecutable ndCuboidStep = new MapReduceExecutable();
+
+        ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : level " + level);
+        StringBuilder cmd = new StringBuilder();
+
+        appendMapReduceParameters(cmd);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, parentPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
+                "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
+        appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + level);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());
+
+        ndCuboidStep.setMapReduceParams(cmd.toString());
+        ndCuboidStep.setMapReduceJobClass(getNDCuboidJob());
+        return ndCuboidStep;
+    }
+
+    private MapReduceExecutable createInMemCubingStep(String jobId, CuboidModeEnum cuboidMode, String cuboidRootPath) {
+        MapReduceExecutable cubeStep = new MapReduceExecutable();
+
+        StringBuilder cmd = new StringBuilder();
+        appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);
+
+        cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);
+
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, getBaseCuboidPath(cuboidRootPath));
+        appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getInMemCuboidPath(cuboidRootPath));
+        appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
+                "Kylin_Cube_Builder_" + seg.getRealization().getName());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());
+
+        cubeStep.setMapReduceParams(cmd.toString());
+        cubeStep.setMapReduceJobClass(InMemCuboidFromBaseCuboidJob.class);
+        cubeStep.setCounterSaveAs(
+                CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
+        return cubeStep;
+    }
+
+    public UpdateCubeInfoAfterOptimizeStep createUpdateCubeInfoAfterOptimizeStep(String jobId) {
+        final UpdateCubeInfoAfterOptimizeStep result = new UpdateCubeInfoAfterOptimizeStep();
+        result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
+
+        CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+        CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+        CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
+
+        return result;
+    }
+
+    protected Class<? extends AbstractHadoopJob> getNDCuboidJob() {
+        return NDCuboidJob.class;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java
new file mode 100644
index 0000000..281cd64
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterCheckpointStep;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.execution.CheckpointExecutable;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.base.Preconditions;
+
+public class BatchOptimizeJobCheckpointBuilder {
+
+    protected static SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
+
+    final protected CubeInstance cube;
+    final protected String submitter;
+
+    private final IMROutput2.IMRBatchOptimizeOutputSide2 outputSide;
+
+    public BatchOptimizeJobCheckpointBuilder(CubeInstance cube, String submitter) {
+        this.cube = cube;
+        this.submitter = submitter;
+
+        Preconditions.checkNotNull(cube.getFirstSegment(), "Cube " + cube + " is empty!!!");
+        this.outputSide = MRUtil.getBatchOptimizeOutputSide2(cube.getFirstSegment());
+    }
+
+    public CheckpointExecutable build() {
+        KylinConfig kylinConfig = cube.getConfig();
+        List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
+                cube.getName());
+        if (projList == null || projList.size() == 0) {
+            throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
+        } else if (projList.size() >= 2) {
+            throw new RuntimeException("Find more than one project containing the cube " + cube.getName()
+                    + ". It does't meet the uniqueness requirement!!! ");
+        }
+
+        CheckpointExecutable checkpointJob = new CheckpointExecutable();
+        checkpointJob.setSubmitter(submitter);
+        CubingExecutableUtil.setCubeName(cube.getName(), checkpointJob.getParams());
+        checkpointJob.setName(
+                cube.getName() + " - OPTIMIZE CHECKPOINT - " + format.format(new Date(System.currentTimeMillis())));
+        checkpointJob.setDeployEnvName(kylinConfig.getDeployEnv());
+        checkpointJob.setProjectName(projList.get(0).getName());
+
+        // Phase 1: Update cube information
+        checkpointJob.addTask(createUpdateCubeInfoAfterCheckpointStep());
+
+        // Phase 2: Garbage collection
+        outputSide.addStepPhase3_Cleanup(checkpointJob);
+
+        return checkpointJob;
+    }
+
+    private UpdateCubeInfoAfterCheckpointStep createUpdateCubeInfoAfterCheckpointStep() {
+        UpdateCubeInfoAfterCheckpointStep result = new UpdateCubeInfoAfterCheckpointStep();
+        result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
+
+        CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
+        return result;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 3cd8931..1fa56c4 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -65,7 +65,7 @@ public class CubingJob extends DefaultChainedExecutable {
     }
 
     public enum CubingJobTypeEnum {
-        BUILD("BUILD"), MERGE("MERGE");
+        BUILD("BUILD"), OPTIMIZE("OPTIMIZE"), MERGE("MERGE");
 
         private final String name;
 
@@ -106,6 +106,10 @@ public class CubingJob extends DefaultChainedExecutable {
         return initCubingJob(seg, CubingJobTypeEnum.BUILD.toString(), submitter, config);
     }
 
+    public static CubingJob createOptimizeJob(CubeSegment seg, String submitter, JobEngineConfig config) {
+        return initCubingJob(seg, CubingJobTypeEnum.OPTIMIZE.toString(), submitter, config);
+    }
+
     public static CubingJob createMergeJob(CubeSegment seg, String submitter, JobEngineConfig config) {
         return initCubingJob(seg, CubingJobTypeEnum.MERGE.toString(), submitter, config);
     }
@@ -135,6 +139,7 @@ public class CubingJob extends DefaultChainedExecutable {
         result.setJobType(jobType);
         CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
         CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+        CubingExecutableUtil.setSegmentName(seg.getName(), result.getParams());
         result.setName(jobType + " CUBE - " + seg.getCubeInstance().getName() + " - " + seg.getName() + " - "
                 + format.format(new Date(System.currentTimeMillis())));
         result.setSubmitter(submitter);

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
index 69bba0a..e33d5c8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 
 public interface IMROutput2 {
@@ -67,7 +68,7 @@ public interface IMROutput2 {
         public void configureJobInput(Job job, String input) throws Exception;
 
         /** Configure the OutputFormat of given job. */
-        public void configureJobOutput(Job job, String output, CubeSegment segment, int level) throws Exception;
+        public void configureJobOutput(Job job, String output, CubeSegment segment, CuboidScheduler cuboidScheduler, int level) throws Exception;
 
     }
 
@@ -113,4 +114,14 @@ public interface IMROutput2 {
         public CubeSegment findSourceSegment(FileSplit fileSplit, CubeInstance cube);
     }
 
+    public IMRBatchOptimizeOutputSide2 getBatchOptimizeOutputSide(CubeSegment seg);
+
+    public interface IMRBatchOptimizeOutputSide2 {
+
+        public void addStepPhase1_CreateHTable(DefaultChainedExecutable jobFlow);
+
+        public void addStepPhase2_BuildCube(DefaultChainedExecutable jobFlow);
+
+        public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c1ed345..27f1d7d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -22,9 +22,11 @@ import java.io.IOException;
 import java.util.List;
 
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
 import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.engine.mr.steps.CalculateStatsFromBaseCuboidJob;
 import org.apache.kylin.engine.mr.steps.CreateDictionaryJob;
 import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
 import org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob;
@@ -47,6 +49,10 @@ public class JobBuilderSupport {
 
     final public static String LayeredCuboidFolderPrefix = "level_";
 
+    final public static String PathNameCuboidBase = "base_cuboid";
+    final public static String PathNameCuboidOld = "old";
+    final public static String PathNameCuboidInMem = "in_memory";
+
     public JobBuilderSupport(CubeSegment seg, String submitter) {
         Preconditions.checkNotNull(seg, "segment cannot be null");
         this.config = new JobEngineConfig(seg.getConfig());
@@ -81,6 +87,31 @@ public class JobBuilderSupport {
         return result;
     }
 
+    public MapReduceExecutable createCalculateStatsFromBaseCuboid(String inputPath, String outputPath) {
+        return createCalculateStatsFromBaseCuboid(inputPath, outputPath, CuboidModeEnum.CURRENT);
+    }
+
+    public MapReduceExecutable createCalculateStatsFromBaseCuboid(String inputPath, String outputPath,
+            CuboidModeEnum cuboidMode) {
+        MapReduceExecutable result = new MapReduceExecutable();
+        result.setName(ExecutableConstants.STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID);
+        result.setMapReduceJobClass(CalculateStatsFromBaseCuboidJob.class);
+        StringBuilder cmd = new StringBuilder();
+        appendMapReduceParameters(cmd);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
+        appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
+        appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT,
+                String.valueOf(config.getConfig().getCubingInMemSamplingPercent()));
+        appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
+                "Calculate_Stats_For_Segment_" + seg.getRealization().getName() + "_Step");
+        appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());
+
+        result.setMapReduceParams(cmd.toString());
+        return result;
+    }
+
     public HadoopShellExecutable createBuildDictionaryStep(String jobId) {
         // base cuboid job
         HadoopShellExecutable buildDictionaryStep = new HadoopShellExecutable();
@@ -176,6 +207,18 @@ public class JobBuilderSupport {
         return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + BatchConstants.CFG_OUTPUT_STATISTICS;
     }
 
+    public String getOptimizationRootPath(String jobId) {
+        return getRealizationRootPath(jobId) + "/optimize";
+    }
+
+    public String getOptimizationStatisticsPath(String jobId) {
+        return getOptimizationRootPath(jobId) + "/statistics";
+    }
+
+    public String getOptimizationCuboidPath(String jobId) {
+        return getOptimizationRootPath(jobId) + "/cuboid/";
+    }
+
     // ============================================================================
     // static methods also shared by other job flow participant
     // ----------------------------------------------------------------------------
@@ -197,11 +240,17 @@ public class JobBuilderSupport {
 
     public static String getCuboidOutputPathsByLevel(String cuboidRootPath, int level) {
         if (level == 0) {
-            return cuboidRootPath + LayeredCuboidFolderPrefix + "base_cuboid";
+            return cuboidRootPath + LayeredCuboidFolderPrefix + PathNameCuboidBase;
         } else {
             return cuboidRootPath + LayeredCuboidFolderPrefix + level + "_cuboid";
         }
     }
 
+    public static String getBaseCuboidPath(String cuboidRootPath) {
+        return cuboidRootPath + PathNameCuboidBase;
+    }
 
+    public static String getInMemCuboidPath(String cuboidRootPath) {
+        return cuboidRootPath + PathNameCuboidInMem;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
index 681c545..74c9b6d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
@@ -48,6 +48,11 @@ public class MRBatchCubingEngine implements IBatchCubingEngine {
     }
 
     @Override
+    public DefaultChainedExecutable createBatchOptimizeJob(CubeSegment optimizeSegment, String submitter) {
+        return new BatchOptimizeJobBuilder2(optimizeSegment, submitter).build();
+    }
+
+    @Override
     public Class<?> getSourceInterface() {
         return IMRInput.class;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
index d9fdcb9..665e791 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
@@ -48,6 +48,11 @@ public class MRBatchCubingEngine2 implements IBatchCubingEngine {
     }
 
     @Override
+    public DefaultChainedExecutable createBatchOptimizeJob(CubeSegment optimizeSegment, String submitter) {
+        return new BatchOptimizeJobBuilder2(optimizeSegment, submitter).build();
+    }
+
+    @Override
     public Class<?> getSourceInterface() {
         return IMRInput.class;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index cbb68d2..f52d64e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -75,6 +75,10 @@ public class MRUtil {
         return SourceFactory.createEngineAdapter(seg, IMRInput.class).getBatchMergeInputSide(seg);
     }
 
+    public static IMROutput2.IMRBatchOptimizeOutputSide2 getBatchOptimizeOutputSide2(CubeSegment seg) {
+        return StorageFactory.createEngineAdapter(seg, IMROutput2.class).getBatchOptimizeOutputSide(seg);
+    }
+    
     // use this method instead of ToolRunner.run() because ToolRunner.run() is not thread-sale
     // Refer to: http://stackoverflow.com/questions/22462665/is-hadoops-toorunner-thread-safe
     public static int runMRJob(Tool tool, String[] args) throws Exception {

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a608c40..7dbae55 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -29,6 +29,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -93,6 +94,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
     protected static final Option OPTION_STATISTICS_ENABLED = OptionBuilder.withArgName(BatchConstants.ARG_STATS_ENABLED).hasArg().isRequired(false).withDescription("Statistics enabled").create(BatchConstants.ARG_STATS_ENABLED);
     protected static final Option OPTION_STATISTICS_OUTPUT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_OUTPUT).hasArg().isRequired(false).withDescription("Statistics output").create(BatchConstants.ARG_STATS_OUTPUT);
     protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_SAMPLING_PERCENT).hasArg().isRequired(false).withDescription("Statistics sampling percentage").create(BatchConstants.ARG_STATS_SAMPLING_PERCENT);
+    protected static final Option OPTION_CUBOID_MODE = OptionBuilder.withArgName(BatchConstants.ARG_CUBOID_MODE)
+            .hasArg().isRequired(false).withDescription("Cuboid Mode").create(BatchConstants.ARG_CUBOID_MODE);
 
     private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
 
@@ -459,7 +462,21 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
         dumpKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
     }
 
+    protected void attachSegmentsMetadataWithDict(List<CubeSegment> segments, Configuration conf) throws IOException {
+        Set<String> dumpList = new LinkedHashSet<>();
+        CubeInstance cube = segments.get(0).getCubeInstance();
+        dumpList.addAll(collectCubeMetadata(cube));
+        for (CubeSegment segment : segments) {
+            dumpList.addAll(segment.getDictionaryPaths());
+        }
+        dumpKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+    }
+
     protected void attachSegmentMetadataWithDict(CubeSegment segment, Configuration conf) throws IOException {
+        attachSegmentMetadata(segment, conf, true, false);
+    }
+
+    protected void attachSegmentMetadataWithAll(CubeSegment segment, Configuration conf) throws IOException {
         attachSegmentMetadata(segment, conf, true, true);
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 74a9d09..eb7673b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -70,6 +70,7 @@ public interface BatchConstants {
      */
     String ARG_INPUT = "input";
     String ARG_OUTPUT = "output";
+    String ARG_CUBOID_MODE = "cuboidMode";
     String ARG_JOB_NAME = "jobname";
     String ARG_CUBING_JOB_ID = "cubingJobId";
     String ARG_CUBE_NAME = "cubename";

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 8f1da6e..a6a9710 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -80,51 +80,52 @@ public class CubeStatsReader {
     final CuboidScheduler cuboidScheduler;
 
     public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
+        this(cubeSegment, cubeSegment.getCuboidScheduler(), kylinConfig);
+    }
+
+    /**
+     * @param cuboidScheduler if it's null, part of it's functions will not be supported
+     */
+    public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig)
+            throws IOException {
         ResourceStore store = ResourceStore.getStore(kylinConfig);
-        cuboidScheduler = cubeSegment.getCuboidScheduler();
         String statsKey = cubeSegment.getStatisticsResourcePath();
         File tmpSeqFile = writeTmpSeqFile(store.getResource(statsKey).inputStream);
-        Reader reader = null;
-
-        try {
-            Configuration hadoopConf = HadoopUtil.getCurrentConfiguration();
-
-            Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath()));
-            Option seqInput = SequenceFile.Reader.file(path);
-            reader = new SequenceFile.Reader(hadoopConf, seqInput);
-
-            int percentage = 100;
-            int mapperNumber = 0;
-            double mapperOverlapRatio = 0;
-            Map<Long, HLLCounter> counterMap = Maps.newHashMap();
-
-            LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf);
-            BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf);
-            while (reader.next(key, value)) {
-                if (key.get() == 0L) {
-                    percentage = Bytes.toInt(value.getBytes());
-                } else if (key.get() == -1) {
-                    mapperOverlapRatio = Bytes.toDouble(value.getBytes());
-                } else if (key.get() == -2) {
-                    mapperNumber = Bytes.toInt(value.getBytes());
-                } else if (key.get() > 0) {
-                    HLLCounter hll = new HLLCounter(kylinConfig.getCubeStatsHLLPrecision());
-                    ByteArray byteArray = new ByteArray(value.getBytes());
-                    hll.readRegisters(byteArray.asBuffer());
-                    counterMap.put(key.get(), hll);
-                }
-            }
-
-            this.seg = cubeSegment;
-            this.samplingPercentage = percentage;
-            this.mapperNumberOfFirstBuild = mapperNumber;
-            this.mapperOverlapRatioOfFirstBuild = mapperOverlapRatio;
-            this.cuboidRowEstimatesHLL = counterMap;
+        Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath()));
+
+        CubeStatsResult cubeStatsResult = new CubeStatsResult();
+        cubeStatsResult.initialize(path, kylinConfig.getCubeStatsHLLPrecision());
+        tmpSeqFile.delete();
+
+        this.seg = cubeSegment;
+        this.cuboidScheduler = cuboidScheduler;
+        this.samplingPercentage = cubeStatsResult.percentage;
+        this.mapperNumberOfFirstBuild = cubeStatsResult.mapperNumber;
+        this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.mapperOverlapRatio;
+        this.cuboidRowEstimatesHLL = cubeStatsResult.counterMap;
+    }
 
-        } finally {
-            IOUtils.closeStream(reader);
-            tmpSeqFile.delete();
-        }
+    /**
+     * Read statistics from
+     * @param path
+     * rather than
+     * @param cubeSegment
+     *
+     * Since the statistics are from
+     * @param path
+     * cuboid scheduler should be provided by default
+     */
+    public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig, Path path)
+            throws IOException {
+        CubeStatsResult cubeStatsResult = new CubeStatsResult();
+        cubeStatsResult.initialize(path, kylinConfig.getCubeStatsHLLPrecision());
+
+        this.seg = cubeSegment;
+        this.cuboidScheduler = cuboidScheduler;
+        this.samplingPercentage = cubeStatsResult.percentage;
+        this.mapperNumberOfFirstBuild = cubeStatsResult.mapperNumber;
+        this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.mapperOverlapRatio;
+        this.cuboidRowEstimatesHLL = cubeStatsResult.counterMap;
     }
 
     private File writeTmpSeqFile(InputStream inputStream) throws IOException {
@@ -140,6 +141,10 @@ public class CubeStatsReader {
         return tempFile;
     }
 
+    public int getSamplingPercentage() {
+        return samplingPercentage;
+    }
+
     public Map<Long, HLLCounter> getCuboidRowEstimatesHLLOrigin() {
         return this.cuboidRowEstimatesHLL;
     }
@@ -253,6 +258,9 @@ public class CubeStatsReader {
 
     //return MB
     public double estimateLayerSize(int level) {
+        if (cuboidScheduler == null) {
+            throw new UnsupportedOperationException("cuboid scheduler is null");
+        }
         List<List<Long>> layeredCuboids = cuboidScheduler.getCuboidsByLayer();
         Map<Long, Double> cuboidSizeMap = getCuboidSizeMap();
         double ret = 0;
@@ -265,11 +273,17 @@ public class CubeStatsReader {
     }
 
     public List<Long> getCuboidsByLayer(int level) {
+        if (cuboidScheduler == null) {
+            throw new UnsupportedOperationException("cuboid scheduler is null");
+        }
         List<List<Long>> layeredCuboids = cuboidScheduler.getCuboidsByLayer();
         return layeredCuboids.get(level);
     }
 
     private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) {
+        if (cuboidScheduler == null) {
+            throw new UnsupportedOperationException("cuboid scheduler is null");
+        }
         long baseCuboid = Cuboid.getBaseCuboidId(seg.getCubeDesc());
         int dimensionCount = Long.bitCount(baseCuboid);
         printCuboidInfoTree(-1L, baseCuboid, cuboidScheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
@@ -317,6 +331,36 @@ public class CubeStatsReader {
         return new DecimalFormat("#.##").format(input);
     }
 
+    private class CubeStatsResult {
+        private int percentage = 100;
+        private double mapperOverlapRatio = 0;
+        private int mapperNumber = 0;
+        Map<Long, HLLCounter> counterMap = Maps.newHashMap();
+
+        void initialize(Path path, int precision) throws IOException {
+            Configuration hadoopConf = HadoopUtil.getCurrentConfiguration();
+            Option seqInput = SequenceFile.Reader.file(path);
+            try (Reader reader = new SequenceFile.Reader(hadoopConf, seqInput)) {
+                LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf);
+                BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf);
+                while (reader.next(key, value)) {
+                    if (key.get() == 0L) {
+                        percentage = Bytes.toInt(value.getBytes());
+                    } else if (key.get() == -1) {
+                        mapperOverlapRatio = Bytes.toDouble(value.getBytes());
+                    } else if (key.get() == -2) {
+                        mapperNumber = Bytes.toInt(value.getBytes());
+                    } else if (key.get() > 0) {
+                        HLLCounter hll = new HLLCounter(precision);
+                        ByteArray byteArray = new ByteArray(value.getBytes());
+                        hll.readRegisters(byteArray.asBuffer());
+                        counterMap.put(key.get(), hll);
+                    }
+                }
+            }
+        }
+    }
+
     public static void main(String[] args) throws IOException {
         System.out.println("CubeStatsReader is used to read cube statistic saved in metadata store");
         KylinConfig config = KylinConfig.getInstanceFromEnv();

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
index ba3f023..649eeb6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -39,20 +39,21 @@ public class CuboidRecommenderUtil {
             return null;
         }
 
-        CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, segment.getConfig());
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, null, segment.getConfig());
         if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
                 || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
             logger.info("Cuboid Statistics is not enabled.");
             return null;
         }
-        long baseCuboid = segment.getCuboidScheduler().getBaseCuboidId();
+        CubeInstance cube = segment.getCubeInstance();
+        long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
         if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
                 || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
             logger.info("Base cuboid count in cuboid statistics is 0.");
             return null;
         }
 
-        String key = segment.getCubeInstance().getName();
+        String key = cube.getName();
         CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
                 cubeStatsReader.getCuboidSizeMap()).build();
         return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(), false);
@@ -81,20 +82,21 @@ public class CuboidRecommenderUtil {
             return null;
         }
 
-        CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, segment.getConfig());
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, null, segment.getConfig());
         if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
                 || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
             logger.info("Cuboid Statistics is not enabled.");
             return null;
         }
-        long baseCuboid = segment.getCuboidScheduler().getBaseCuboidId();
+        CubeInstance cube = segment.getCubeInstance();
+        long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
         if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
                 || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
             logger.info("Base cuboid count in cuboid statistics is 0.");
             return null;
         }
 
-        String key = segment.getCubeInstance().getName() + "-" + segment.getName();
+        String key = cube.getName() + "-" + segment.getName();
         CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
                 cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
                         .setRollingUpCountSourceMap(rollingUpCountSourceMap,

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidSchedulerUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidSchedulerUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidSchedulerUtil.java
new file mode 100644
index 0000000..d684c04
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidSchedulerUtil.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
+import org.apache.kylin.cube.cuboid.TreeCuboidSchedulerManager;
+
+public class CuboidSchedulerUtil {
+
+    public static CuboidScheduler getCuboidSchedulerByMode(CubeSegment segment, String cuboidModeName) {
+        return getCuboidSchedulerByMode(segment, segment.getCubeInstance().getCuboidsByMode(cuboidModeName));
+    }
+
+    public static CuboidScheduler getCuboidSchedulerByMode(CubeSegment segment, CuboidModeEnum cuboidMode) {
+        return getCuboidSchedulerByMode(segment, segment.getCubeInstance().getCuboidsByMode(cuboidMode));
+    }
+
+    public static CuboidScheduler getCuboidSchedulerByMode(CubeSegment segment, Set<Long> cuboidSet) {
+        CuboidScheduler cuboidScheduler;
+        try {
+            cuboidScheduler = TreeCuboidSchedulerManager.getInstance().getTreeCuboidScheduler(segment.getCubeDesc(), //
+                    CuboidStatsReaderUtil.readCuboidStatsFromSegment(cuboidSet, segment));
+        } catch (IOException e) {
+            throw new RuntimeException("Fail to cube stats for segment" + segment + " due to " + e);
+        }
+
+        if (cuboidScheduler == null) {
+            cuboidScheduler = new DefaultCuboidScheduler(segment.getCubeDesc());
+        }
+        return cuboidScheduler;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/faf08625/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
index aaf9aa3..bfb37ce 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -74,7 +74,7 @@ public class CuboidStatsReaderUtil {
         Map<Long, HLLCounter> cuboidHLLMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
         Map<Long, Double> sizeMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
         for (CubeSegment pSegment : segmentList) {
-            CubeStatsReader pReader = new CubeStatsReader(pSegment, pSegment.getConfig());
+            CubeStatsReader pReader = new CubeStatsReader(pSegment, null, pSegment.getConfig());
             Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowEstimatesHLLOrigin();
             if (pHLLMap == null || pHLLMap.isEmpty()) {
                 logger.info("Cuboid Statistics for segment " + pSegment.getName() + " is not enabled.");
@@ -113,7 +113,7 @@ public class CuboidStatsReaderUtil {
             return null;
         }
 
-        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, cubeSegment.getConfig());
+        CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, null, cubeSegment.getConfig());
         if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
                 || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
             logger.info("Cuboid Statistics is not enabled.");
@@ -132,4 +132,5 @@ public class CuboidStatsReaderUtil {
         }
         return cuboidsWithStats;
     }
+
 }


[13/22] kylin git commit: APACHE-KYLIN-2732: Introduce base cuboid as a new input for cubing job

Posted by nj...@apache.org.
APACHE-KYLIN-2732: Introduce base cuboid as a new input for cubing job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/386a2ff7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/386a2ff7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/386a2ff7

Branch: refs/heads/yaho-cube-planner
Commit: 386a2ff7f47faca6a402c1afcfb0ab42c06e59ae
Parents: 05ce035
Author: Zhong <nj...@apache.org>
Authored: Mon Aug 28 11:34:17 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:33:04 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/CubeInstance.java     |   2 +-
 .../cube/cuboid/TreeCuboidSchedulerManager.java | 102 +++++++++
 .../inmemcubing/AbstractInMemCubeBuilder.java   |  18 +-
 .../ConsumeBlockingQueueController.java         |  84 ++++++++
 .../cube/inmemcubing/DoggedCubeBuilder.java     | 117 ++--------
 .../cube/inmemcubing/InMemCubeBuilder.java      |  87 ++------
 .../InMemCubeBuilderInputConverter.java         | 149 -------------
 .../kylin/cube/inmemcubing/InputConverter.java  |  69 ++++++
 .../cube/inmemcubing/InputConverterUnit.java    |  33 +++
 .../InputConverterUnitForBaseCuboid.java        |  55 +++++
 .../InputConverterUnitForRawData.java           | 165 ++++++++++++++
 .../RecordConsumeBlockingQueueController.java   |  91 ++++++++
 .../org/apache/kylin/gridtable/GTRecord.java    |   5 +
 .../engine/mr/common/AbstractHadoopJob.java     |  12 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   1 +
 .../engine/mr/steps/InMemCuboidMapper.java      | 129 +++--------
 .../engine/mr/steps/InMemCuboidMapperBase.java  | 214 +++++++++++++++++++
 .../apache/kylin/engine/spark/SparkCubing.java  | 164 +++++++-------
 .../ITDoggedCubeBuilderStressTest.java          |   3 +-
 .../inmemcubing/ITDoggedCubeBuilderTest.java    |   3 +-
 .../inmemcubing/ITInMemCubeBuilderTest.java     |  14 +-
 21 files changed, 1008 insertions(+), 509 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index d217dcd..05fb5be 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -330,7 +330,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
     }
 
     public Set<Long> getCuboidsByMode(String cuboidModeName) {
-        return getCuboidsByMode(CuboidModeEnum.getByModeName(cuboidModeName));
+        return getCuboidsByMode(cuboidModeName == null ? null : CuboidModeEnum.getByModeName(cuboidModeName));
     }
 
     public Set<Long> getCuboidsByMode(CuboidModeEnum cuboidMode) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
new file mode 100644
index 0000000..5e8d965
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidSchedulerManager.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class TreeCuboidSchedulerManager {
+    private static ConcurrentMap<String, TreeCuboidScheduler> cache = Maps.newConcurrentMap();
+
+    private class TreeCuboidSchedulerSyncListener extends Broadcaster.Listener {
+        @Override
+        public void onClearAll(Broadcaster broadcaster) throws IOException {
+            cache.clear();
+        }
+
+        @Override
+        public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey)
+                throws IOException {
+            cache.remove(cacheKey);
+        }
+    }
+
+    public TreeCuboidSchedulerManager() {
+        Broadcaster.getInstance(KylinConfig.getInstanceFromEnv())
+                .registerListener(new TreeCuboidSchedulerSyncListener(), "cube");
+    }
+
+    private static TreeCuboidSchedulerManager instance = new TreeCuboidSchedulerManager();
+
+    public static TreeCuboidSchedulerManager getInstance() {
+        return instance;
+    }
+
+    /**
+     *
+     * @param cubeName
+     * @return null if the cube has no pre-built cuboids
+     */
+    public static TreeCuboidScheduler getTreeCuboidScheduler(String cubeName) {
+        TreeCuboidScheduler result = cache.get(cubeName);
+        if (result == null) {
+            CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+            CubeInstance cubeInstance = cubeManager.getCube(cubeName);
+            if (cubeInstance == null) {
+                return null;
+            }
+            TreeCuboidScheduler treeCuboidScheduler = getTreeCuboidScheduler(cubeInstance.getDescriptor(),
+                    cubeManager.getCube(cubeName).getCuboids());
+            if (treeCuboidScheduler == null) {
+                return null;
+            }
+            cache.put(cubeName, treeCuboidScheduler);
+            result = treeCuboidScheduler;
+        }
+        return result;
+    }
+
+    public static TreeCuboidScheduler getTreeCuboidScheduler(CubeDesc cubeDesc, Map<Long, Long> cuboidsWithRowCnt) {
+        if (cuboidsWithRowCnt == null || cuboidsWithRowCnt.isEmpty()) {
+            return null;
+        }
+        return getTreeCuboidScheduler(cubeDesc, Lists.newArrayList(cuboidsWithRowCnt.keySet()), cuboidsWithRowCnt);
+    }
+
+    public static TreeCuboidScheduler getTreeCuboidScheduler(CubeDesc cubeDesc, List<Long> cuboidIds,
+            Map<Long, Long> cuboidsWithRowCnt) {
+        if (cuboidIds == null || cuboidsWithRowCnt == null) {
+            return null;
+        }
+        TreeCuboidScheduler treeCuboidScheduler = new TreeCuboidScheduler(cubeDesc, cuboidIds,
+                new TreeCuboidScheduler.CuboidCostComparator(cuboidsWithRowCnt));
+        return treeCuboidScheduler;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index ae38261..df1fa7a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -19,7 +19,6 @@
 package org.apache.kylin.cube.inmemcubing;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 
@@ -51,11 +50,6 @@ abstract public class AbstractInMemCubeBuilder {
     protected int taskThreadCount = 1;
     protected int reserveMemoryMB = 100;
 
-    //    @Deprecated
-    //    public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-    //        this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap);
-    //    }
-
     protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
             Map<TblColRef, Dictionary<String>> dictionaryMap) {
         if (cuboidScheduler == null)
@@ -83,12 +77,17 @@ abstract public class AbstractInMemCubeBuilder {
         return this.reserveMemoryMB;
     }
 
-    public Runnable buildAsRunnable(final BlockingQueue<List<String>> input, final ICuboidWriter output) {
+    public Runnable buildAsRunnable(final BlockingQueue<String[]> input, final ICuboidWriter output) {
+        return buildAsRunnable(input, new InputConverterUnitForRawData(cubeDesc, flatDesc, dictionaryMap), output);
+    }
+
+    public <T> Runnable buildAsRunnable(final BlockingQueue<T> input, final InputConverterUnit<T> inputConverterUnit,
+            final ICuboidWriter output) {
         return new Runnable() {
             @Override
             public void run() {
                 try {
-                    build(input, output);
+                    build(input, inputConverterUnit, output);
                 } catch (IOException e) {
                     throw new RuntimeException(e);
                 }
@@ -96,7 +95,8 @@ abstract public class AbstractInMemCubeBuilder {
         };
     }
 
-    abstract public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException;
+    abstract public <T> void build(BlockingQueue<T> input, InputConverterUnit<T> inputConverterUnit,
+            ICuboidWriter output) throws IOException;
 
     protected void outputCuboid(long cuboidId, GridTable gridTable, ICuboidWriter output) throws IOException {
         long startTime = System.currentTimeMillis();

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConsumeBlockingQueueController.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConsumeBlockingQueueController.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConsumeBlockingQueueController.java
new file mode 100644
index 0000000..a9e55f7
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConsumeBlockingQueueController.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.inmemcubing;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Lists;
+
+public class ConsumeBlockingQueueController<T> implements Iterator<T> {
+    public final static int DEFAULT_BATCH_SIZE = 1000;
+
+    private volatile boolean hasException = false;
+    private final BlockingQueue<T> input;
+    private final int batchSize;
+    private final List<T> batchBuffer;
+    private Iterator<T> internalIT;
+
+    private AtomicInteger outputRowCount = new AtomicInteger();
+
+    public ConsumeBlockingQueueController(BlockingQueue<T> input, int batchSize) {
+        this.input = input;
+        this.batchSize = batchSize;
+        this.batchBuffer = Lists.newArrayListWithExpectedSize(batchSize);
+        this.internalIT = batchBuffer.iterator();
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (hasException) {
+            return false;
+        }
+        if (internalIT.hasNext()) {
+            return true;
+        } else {
+            batchBuffer.clear();
+            try {
+                batchBuffer.add(input.take());
+                outputRowCount.incrementAndGet();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            outputRowCount.addAndGet(input.drainTo(batchBuffer, batchSize - 1));
+            internalIT = batchBuffer.iterator();
+        }
+        return true;
+    }
+
+    @Override
+    public T next() {
+        return internalIT.next();
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    public void findException() {
+        hasException = true;
+    }
+
+    public long getOutputRowCount() {
+        return outputRowCount.get();
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index dd92a2b..ccd7137 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -20,21 +20,17 @@ package org.apache.kylin.cube.inmemcubing;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
-import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.TimeUnit;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequestBuilder;
@@ -55,7 +51,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
     private static Logger logger = LoggerFactory.getLogger(DoggedCubeBuilder.class);
 
     private int splitRowThreshold = Integer.MAX_VALUE;
-    private int unitRows = 1000;
+    private int unitRows = ConsumeBlockingQueueController.DEFAULT_BATCH_SIZE;
 
     public DoggedCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
             Map<TblColRef, Dictionary<String>> dictionaryMap) {
@@ -72,8 +68,9 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     @Override
-    public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
-        new BuildOnce().build(input, output);
+    public <T> void build(BlockingQueue<T> input, InputConverterUnit<T> inputConverterUnit, ICuboidWriter output)
+            throws IOException {
+        new BuildOnce().build(input, inputConverterUnit, output);
     }
 
     private class BuildOnce {
@@ -81,7 +78,8 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
         BuildOnce() {
         }
 
-        public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
+        public <T> void build(BlockingQueue<T> input, InputConverterUnit<T> inputConverterUnit, ICuboidWriter output)
+                throws IOException {
             final List<SplitThread> splits = new ArrayList<SplitThread>();
             final Merger merger = new Merger();
 
@@ -89,32 +87,23 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
             logger.info("Dogged Cube Build start");
 
             try {
-                SplitThread last = null;
-                boolean eof = false;
+                while (true) {
+                    SplitThread last = new SplitThread(splits.size() + 1, RecordConsumeBlockingQueueController
+                            .getQueueController(inputConverterUnit, input, unitRows));
+                    splits.add(last);
 
-                while (!eof) {
+                    last.start();
+                    logger.info("Split #" + splits.size() + " kickoff");
 
-                    if (last != null && shouldCutSplit(splits)) {
-                        cutSplit(last);
-                        last = null;
-                    }
+                    // Build splits sequentially
+                    last.join();
 
                     checkException(splits);
-
-                    if (last == null) {
-                        last = new SplitThread();
-                        splits.add(last);
-                        last.start();
-                        logger.info("Split #" + splits.size() + " kickoff");
+                    if (last.inputController.ifEnd()) {
+                        break;
                     }
-
-                    eof = feedSomeInput(input, last, unitRows);
                 }
 
-                for (SplitThread split : splits) {
-                    split.join();
-                }
-                checkException(splits);
                 logger.info("Dogged Cube Build splits complete, took " + (System.currentTimeMillis() - start) + " ms");
 
                 merger.mergeAndOutput(splits, output);
@@ -202,81 +191,18 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
                 throw new IOException(errors.size() + " exceptions during in-mem cube build, cause set to the first, check log for more", errors.get(0));
             }
         }
-
-        private boolean feedSomeInput(BlockingQueue<List<String>> input, SplitThread split, int n) {
-            try {
-                int i = 0;
-                while (i < n) {
-                    List<String> record = input.take();
-                    i++;
-
-                    while (split.inputQueue.offer(record, 1, TimeUnit.SECONDS) == false) {
-                        if (split.exception != null)
-                            return true; // got some error
-                    }
-                    split.inputRowCount++;
-
-                    if (record == null || record.isEmpty()) {
-                        return true;
-                    }
-                }
-                return false;
-
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                throw new RuntimeException(e);
-            }
-        }
-
-        private void cutSplit(SplitThread last) {
-            try {
-                // signal the end of input
-                while (last.isAlive()) {
-                    if (last.inputQueue.offer(Collections.<String> emptyList())) {
-                        break;
-                    }
-                    Thread.sleep(1000);
-                }
-
-                // wait cuboid build done
-                last.join();
-
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                throw new RuntimeException(e);
-            }
-        }
-
-        private boolean shouldCutSplit(List<SplitThread> splits) {
-            int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
-            int nSplit = splits.size();
-            long splitRowCount = nSplit == 0 ? 0 : splits.get(nSplit - 1).inputRowCount;
-
-            logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold");
-
-            if (splitRowCount >= splitRowThreshold) {
-                logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold);
-                return true;
-            }
-
-            if (systemAvailMB <= reserveMemoryMB * 1.5) {
-                logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + "*1.5 MB");
-                return true;
-            }
-
-            return false;
-        }
     }
 
     private class SplitThread extends Thread {
-        final BlockingQueue<List<String>> inputQueue = new ArrayBlockingQueue<List<String>>(16);
+        final RecordConsumeBlockingQueueController<?> inputController;
         final InMemCubeBuilder builder;
 
         ConcurrentNavigableMap<Long, CuboidResult> buildResult;
-        long inputRowCount = 0;
         RuntimeException exception;
 
-        public SplitThread() {
+        public SplitThread(final int num, final RecordConsumeBlockingQueueController<?> inputController) {
+            super("SplitThread" + num);
+            this.inputController = inputController;
             this.builder = new InMemCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap);
             this.builder.setConcurrentThreads(taskThreadCount);
             this.builder.setReserveMemoryMB(reserveMemoryMB);
@@ -285,12 +211,13 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
         @Override
         public void run() {
             try {
-                buildResult = builder.build(inputQueue);
+                buildResult = builder.build(inputController);
             } catch (Exception e) {
                 if (e instanceof RuntimeException)
                     this.exception = (RuntimeException) e;
                 else
                     this.exception = new RuntimeException(e);
+                inputController.findException();
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 684c26b..f63b53f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -87,6 +87,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     private CuboidResult baseResult;
     private Object[] totalSumForSanityCheck;
     private ICuboidCollector resultCollector;
+    private boolean ifBaseCuboidCollected = true;
 
     public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
             Map<TblColRef, Dictionary<String>> dictionaryMap) {
@@ -121,8 +122,10 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     @Override
-    public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
-        ConcurrentNavigableMap<Long, CuboidResult> result = build(input);
+    public <T> void build(BlockingQueue<T> input, InputConverterUnit<T> inputConverterUnit, ICuboidWriter output)
+            throws IOException {
+        ConcurrentNavigableMap<Long, CuboidResult> result = build(
+                RecordConsumeBlockingQueueController.getQueueController(inputConverterUnit, input));
         try {
             for (CuboidResult cuboidResult : result.values()) {
                 outputCuboid(cuboidResult.cuboidId, cuboidResult.table, output);
@@ -133,7 +136,11 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
         }
     }
 
-    public ConcurrentNavigableMap<Long, CuboidResult> build(BlockingQueue<List<String>> input) throws IOException {
+    public <T> ConcurrentNavigableMap<Long, CuboidResult> build(RecordConsumeBlockingQueueController<T> input)
+            throws IOException {
+        if (input.inputConverterUnit instanceof InputConverterUnitForBaseCuboid) {
+            ifBaseCuboidCollected = false;
+        }
         final ConcurrentNavigableMap<Long, CuboidResult> result = new ConcurrentSkipListMap<Long, CuboidResult>();
         build(input, new ICuboidCollector() {
             @Override
@@ -150,7 +157,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
         void collect(CuboidResult result);
     }
 
-    private void build(BlockingQueue<List<String>> input, ICuboidCollector collector) throws IOException {
+    private <T> void build(RecordConsumeBlockingQueueController<T> input, ICuboidCollector collector)
+            throws IOException {
         long startTime = System.currentTimeMillis();
         logger.info("In Mem Cube Build start, " + cubeDesc.getName());
 
@@ -326,7 +334,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
         memBudget = new MemoryBudgetController(budget);
     }
 
-    private CuboidResult createBaseCuboid(BlockingQueue<List<String>> input) throws IOException {
+    private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException {
         long startTime = System.currentTimeMillis();
         logger.info("Calculating base cuboid " + baseCuboidId);
 
@@ -356,10 +364,15 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
         int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB);
         logger.info("Wild estimate of base aggr cache is " + mbEstimateBaseAggrCache + " MB");
 
-        return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0);
+        return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, ifBaseCuboidCollected);
     }
 
     private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent, int aggrCacheMB) {
+        return updateCuboidResult(cuboidId, table, nRows, timeSpent, aggrCacheMB, true);
+    }
+
+    private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent, int aggrCacheMB,
+            boolean ifCollect) {
         if (aggrCacheMB <= 0 && baseResult != null) {
             aggrCacheMB = (int) Math.round(//
                     (DERIVE_AGGR_CACHE_CONSTANT_FACTOR + DERIVE_AGGR_CACHE_VARIABLE_FACTOR * nRows / baseResult.nRows) //
@@ -369,7 +382,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
         CuboidResult result = new CuboidResult(cuboidId, table, nRows, timeSpent, aggrCacheMB);
         taskCuboidCompleted.incrementAndGet();
 
-        resultCollector.collect(result);
+        if (ifCollect) {
+            resultCollector.collect(result);
+        }
         return result;
     }
 
@@ -508,62 +523,4 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
             return comp < 0 ? -1 : (comp > 0 ? 1 : 0);
         }
     }
-
-    // ============================================================================
-
-    private class InputConverter implements IGTScanner {
-        GTInfo info;
-        GTRecord record;
-        BlockingQueue<List<String>> input;
-        final InMemCubeBuilderInputConverter inMemCubeBuilderInputConverter;
-
-        public InputConverter(GTInfo info, BlockingQueue<List<String>> input) {
-            this.info = info;
-            this.input = input;
-            this.record = new GTRecord(info);
-            this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, flatDesc, dictionaryMap, info);
-        }
-
-        @Override
-        public Iterator<GTRecord> iterator() {
-            return new Iterator<GTRecord>() {
-
-                List<String> currentObject = null;
-
-                @Override
-                public boolean hasNext() {
-                    try {
-                        currentObject = input.take();
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                        throw new RuntimeException(e);
-                    }
-                    return currentObject != null && currentObject.size() > 0;
-                }
-
-                @Override
-                public GTRecord next() {
-                    if (currentObject.size() == 0)
-                        throw new IllegalStateException();
-
-                    inMemCubeBuilderInputConverter.convert(currentObject, record);
-                    return record;
-                }
-
-                @Override
-                public void remove() {
-                    throw new UnsupportedOperationException();
-                }
-            };
-        }
-
-        @Override
-        public void close() throws IOException {
-        }
-
-        @Override
-        public GTInfo getInfo() {
-            return info;
-        }
-    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
deleted file mode 100644
index 6dd20d8..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.cube.inmemcubing;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
-import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class InMemCubeBuilderInputConverter {
-
-    @SuppressWarnings("unused")
-    private static final Logger logger = LoggerFactory.getLogger(InMemCubeBuilderInputConverter.class);
-    
-    public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
-
-    private final CubeJoinedFlatTableEnrich flatDesc;
-    private final MeasureDesc[] measureDescs;
-    private final MeasureIngester<?>[] measureIngesters;
-    private final int measureCount;
-    private final Map<TblColRef, Dictionary<String>> dictionaryMap;
-    private final GTInfo gtInfo;
-    protected List<byte[]> nullBytes;
-
-    public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) {
-        this.gtInfo = gtInfo;
-        this.flatDesc = new CubeJoinedFlatTableEnrich(flatDesc, cubeDesc);
-        this.measureCount = cubeDesc.getMeasures().size();
-        this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
-        this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
-        this.dictionaryMap = dictionaryMap;
-        initNullBytes(cubeDesc);
-    }
-
-    public final GTRecord convert(List<String> row) {
-        final GTRecord record = new GTRecord(gtInfo);
-        convert(row, record);
-        return record;
-    }
-
-    public final void convert(List<String> row, GTRecord record) {
-        Object[] dimensions = buildKey(row);
-        Object[] metricsValues = buildValue(row);
-        Object[] recordValues = new Object[dimensions.length + metricsValues.length];
-        System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
-        System.arraycopy(metricsValues, 0, recordValues, dimensions.length, metricsValues.length);
-        record.setValues(recordValues);
-    }
-
-    private Object[] buildKey(List<String> row) {
-        int keySize = flatDesc.getRowKeyColumnIndexes().length;
-        Object[] key = new Object[keySize];
-
-        for (int i = 0; i < keySize; i++) {
-            key[i] = row.get(flatDesc.getRowKeyColumnIndexes()[i]);
-            if (key[i] != null && isNull(Bytes.toBytes((String) key[i]))) {
-                key[i] = null;
-            }
-        }
-
-        return key;
-    }
-
-    private Object[] buildValue(List<String> row) {
-        Object[] values = new Object[measureCount];
-        for (int i = 0; i < measureCount; i++) {
-            values[i] = buildValueOf(i, row);
-        }
-        return values;
-    }
-
-    private Object buildValueOf(int idxOfMeasure, List<String> row) {
-        MeasureDesc measure = measureDescs[idxOfMeasure];
-        FunctionDesc function = measure.getFunction();
-        int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];
-
-        int paramCount = function.getParameterCount();
-        String[] inputToMeasure = new String[paramCount];
-
-        // pick up parameter values
-        ParameterDesc param = function.getParameter();
-        int paramColIdx = 0; // index among parameters of column type
-        for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
-            String value;
-            if (function.isCount()) {
-                value = "1";
-            } else if (param.isColumnType()) {
-                value = row.get(colIdxOnFlatTable[paramColIdx++]);
-            } else {
-                value = param.getValue();
-            }
-            inputToMeasure[i] = value;
-        }
-
-        return measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
-    }
-
-    private void initNullBytes(CubeDesc cubeDesc) {
-        nullBytes = Lists.newArrayList();
-        nullBytes.add(HIVE_NULL);
-        String[] nullStrings = cubeDesc.getNullStrings();
-        if (nullStrings != null) {
-            for (String s : nullStrings) {
-                nullBytes.add(Bytes.toBytes(s));
-            }
-        }
-    }
-
-    private boolean isNull(byte[] v) {
-        for (byte[] nullByte : nullBytes) {
-            if (Bytes.equals(v, nullByte))
-                return true;
-        }
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverter.java
new file mode 100644
index 0000000..664f784
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverter.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.inmemcubing;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.IGTScanner;
+
+public class InputConverter<T> implements IGTScanner {
+    private GTInfo info;
+    private GTRecord record;
+    private RecordConsumeBlockingQueueController<T> inputController;
+
+    public InputConverter(GTInfo info, RecordConsumeBlockingQueueController<T> inputController) {
+        this.info = info;
+        this.inputController = inputController;
+        this.record = new GTRecord(info);
+    }
+
+    @Override
+    public Iterator<GTRecord> iterator() {
+        return new Iterator<GTRecord>() {
+
+            @Override
+            public boolean hasNext() {
+                return inputController.hasNext();
+            }
+
+            @Override
+            public GTRecord next() {
+                inputController.inputConverterUnit.convert(inputController.next(), record);
+                return record;
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    @Override
+    public GTInfo getInfo() {
+        return info;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnit.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnit.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnit.java
new file mode 100644
index 0000000..fe32937
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnit.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.inmemcubing;
+
+import org.apache.kylin.gridtable.GTRecord;
+
+public interface InputConverterUnit<T> {
+    public void convert(T currentObject, GTRecord record);
+
+    public boolean ifEnd(T currentObject);
+
+    public boolean ifCut(T currentObject);
+
+    public T getEmptyUnit();
+
+    public T getCutUnit();
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForBaseCuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForBaseCuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForBaseCuboid.java
new file mode 100644
index 0000000..1bf02ed
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForBaseCuboid.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.inmemcubing;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.gridtable.GTRecord;
+
+public class InputConverterUnitForBaseCuboid implements InputConverterUnit<ByteArray> {
+
+    public static final ByteArray EMPTY_ROW = ByteArray.EMPTY;
+    public static final ByteArray CUT_ROW = new ByteArray(0);
+
+    public void convert(ByteArray currentObject, GTRecord record) {
+        record.loadColumns(currentObject.asBuffer());
+    }
+
+    public boolean ifEnd(ByteArray currentObject) {
+        if (currentObject == null) {
+            return false;
+        }
+        return currentObject == EMPTY_ROW;
+    }
+
+    public ByteArray getEmptyUnit() {
+        return EMPTY_ROW;
+    }
+
+    public ByteArray getCutUnit() {
+        return CUT_ROW;
+    }
+
+    @Override
+    public boolean ifCut(ByteArray currentObject) {
+        if (currentObject == null) {
+            return false;
+        }
+        return currentObject == CUT_ROW;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForRawData.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForRawData.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForRawData.java
new file mode 100644
index 0000000..465cb3b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InputConverterUnitForRawData.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.cube.inmemcubing;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ */
+public class InputConverterUnitForRawData implements InputConverterUnit<String[]> {
+
+    @SuppressWarnings("unused")
+    private static final Logger logger = LoggerFactory.getLogger(InputConverterUnitForRawData.class);
+    
+    public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
+    public static final String[] EMPTY_ROW = new String[0];
+    public static final String[] CUT_ROW = { "" };
+
+    private final CubeJoinedFlatTableEnrich flatDesc;
+    private final MeasureDesc[] measureDescs;
+    private final MeasureIngester<?>[] measureIngesters;
+    private final int measureCount;
+    private final Map<TblColRef, Dictionary<String>> dictionaryMap;
+    protected List<byte[]> nullBytes;
+
+    public InputConverterUnitForRawData(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        this.flatDesc = new CubeJoinedFlatTableEnrich(flatDesc, cubeDesc);
+        this.measureCount = cubeDesc.getMeasures().size();
+        this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
+        this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+        this.dictionaryMap = dictionaryMap;
+        initNullBytes(cubeDesc);
+    }
+
+    public final void convert(String[] row, GTRecord record) {
+        Object[] dimensions = buildKey(row);
+        Object[] metricsValues = buildValue(row);
+        Object[] recordValues = new Object[dimensions.length + metricsValues.length];
+        System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
+        System.arraycopy(metricsValues, 0, recordValues, dimensions.length, metricsValues.length);
+        record.setValues(recordValues);
+    }
+
+    public boolean ifEnd(String[] currentObject) {
+        if (currentObject == null) {
+            return false;
+        }
+        return currentObject == EMPTY_ROW;
+    }
+
+    public boolean ifCut(String[] currentObject) {
+        if (currentObject == null) {
+            return false;
+        }
+        return currentObject == CUT_ROW;
+    }
+
+    public String[] getEmptyUnit() {
+        return EMPTY_ROW;
+    }
+
+    public String[] getCutUnit() {
+        return CUT_ROW;
+    }
+
+    private Object[] buildKey(String[] row) {
+        int keySize = flatDesc.getRowKeyColumnIndexes().length;
+        Object[] key = new Object[keySize];
+
+        for (int i = 0; i < keySize; i++) {
+            key[i] = row[flatDesc.getRowKeyColumnIndexes()[i]];
+            if (key[i] != null && isNull(Bytes.toBytes((String) key[i]))) {
+                key[i] = null;
+            }
+        }
+
+        return key;
+    }
+
+    private Object[] buildValue(String[] row) {
+        Object[] values = new Object[measureCount];
+        for (int i = 0; i < measureCount; i++) {
+            values[i] = buildValueOf(i, row);
+        }
+        return values;
+    }
+
+    private Object buildValueOf(int idxOfMeasure, String[] row) {
+        MeasureDesc measure = measureDescs[idxOfMeasure];
+        FunctionDesc function = measure.getFunction();
+        int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];
+
+        int paramCount = function.getParameterCount();
+        String[] inputToMeasure = new String[paramCount];
+
+        // pick up parameter values
+        ParameterDesc param = function.getParameter();
+        int paramColIdx = 0; // index among parameters of column type
+        for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
+            String value;
+            if (function.isCount()) {
+                value = "1";
+            } else if (param.isColumnType()) {
+                value = row[colIdxOnFlatTable[paramColIdx++]];
+            } else {
+                value = param.getValue();
+            }
+            inputToMeasure[i] = value;
+        }
+
+        return measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
+    }
+
+    private void initNullBytes(CubeDesc cubeDesc) {
+        nullBytes = Lists.newArrayList();
+        nullBytes.add(HIVE_NULL);
+        String[] nullStrings = cubeDesc.getNullStrings();
+        if (nullStrings != null) {
+            for (String s : nullStrings) {
+                nullBytes.add(Bytes.toBytes(s));
+            }
+        }
+    }
+
+    private boolean isNull(byte[] v) {
+        for (byte[] nullByte : nullBytes) {
+            if (Bytes.equals(v, nullByte))
+                return true;
+        }
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/RecordConsumeBlockingQueueController.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/RecordConsumeBlockingQueueController.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/RecordConsumeBlockingQueueController.java
new file mode 100644
index 0000000..49cbe1f
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/RecordConsumeBlockingQueueController.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.inmemcubing;
+
+import java.util.concurrent.BlockingQueue;
+
+public class RecordConsumeBlockingQueueController<T> extends ConsumeBlockingQueueController<T> {
+
+    public final InputConverterUnit<T> inputConverterUnit;
+
+    private RecordConsumeBlockingQueueController(InputConverterUnit<T> inputConverterUnit, BlockingQueue<T> input, int batchSize) {
+        super(input, batchSize);
+        this.inputConverterUnit = inputConverterUnit;
+    }
+   
+    private T currentObject = null;
+    private volatile boolean ifEnd = false;
+    private volatile boolean cut = false;
+    private long outputRowCountCut = 0L;
+
+    @Override
+    public boolean hasNext() {
+        if (currentObject != null) {
+            return hasNext(currentObject);
+        }
+        if (!super.hasNext()) {
+            return false;
+        }
+        currentObject = super.next();
+        return hasNext(currentObject);
+    }
+
+    @Override
+    public T next() {
+        if (ifEnd())
+            throw new IllegalStateException();
+
+        T result = currentObject;
+        currentObject = null;
+        return result;
+    }
+
+    public boolean ifEnd() {
+        return ifEnd;
+    }
+
+    private boolean hasNext(T object) {
+        if (inputConverterUnit.ifEnd(object)) {
+            ifEnd = true;
+            return false;
+        }else if(cut){
+            return false;
+        }else if(inputConverterUnit.ifCut(object)){
+            return false;
+        }
+        return true;
+    }
+    
+    public static <T> RecordConsumeBlockingQueueController<T> getQueueController(InputConverterUnit<T> inputConverterUnit, BlockingQueue<T> input){
+        return new RecordConsumeBlockingQueueController<>(inputConverterUnit, input, DEFAULT_BATCH_SIZE);
+    }
+    
+    public static <T> RecordConsumeBlockingQueueController<T> getQueueController(InputConverterUnit<T> inputConverterUnit, BlockingQueue<T> input, int batchSize){
+        return new RecordConsumeBlockingQueueController<>(inputConverterUnit, input, batchSize);
+    }
+
+    public void forceCutPipe() {
+        cut = true;
+        outputRowCountCut = getOutputRowCount();
+    }
+
+    public long getOutputRowCountAfterCut() {
+        return getOutputRowCount() - outputRowCountCut;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 3e62ea7..ab02030 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -281,6 +281,11 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
         loadColumns(info.colBlocks[c], buf);
     }
 
+    /** change pointers to point to data in given buffer, UNLIKE deserialize */
+    public void loadColumns(ByteBuffer buf) {
+        loadColumns(info.colAll, buf);
+    }
+
     /**
      * Change pointers to point to data in given buffer, UNLIKE deserialize
      * @param selectedCols positions of column to load

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index f9d9808..a608c40 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -460,9 +460,19 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
     }
 
     protected void attachSegmentMetadataWithDict(CubeSegment segment, Configuration conf) throws IOException {
+        attachSegmentMetadata(segment, conf, true, true);
+    }
+
+    protected void attachSegmentMetadata(CubeSegment segment, Configuration conf, boolean ifDictIncluded,
+            boolean ifStatsIncluded) throws IOException {
         Set<String> dumpList = new LinkedHashSet<>();
         dumpList.addAll(collectCubeMetadata(segment.getCubeInstance()));
-        dumpList.addAll(segment.getDictionaryPaths());
+        if (ifDictIncluded) {
+            dumpList.addAll(segment.getDictionaryPaths());
+        }
+        if (ifStatsIncluded) {
+            dumpList.add(segment.getStatisticsResourcePath());
+        }
         dumpKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0cb23ac..74a9d09 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -29,6 +29,7 @@ public interface BatchConstants {
      * ConFiGuration entry names for MR jobs
      */
 
+    String CFG_CUBOID_MODE = "cuboid.mode";
     String CFG_CUBE_NAME = "cube.name";
     String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
     String CFG_CUBE_SEGMENT_ID = "cube.segment.id";

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index 5511414..30f4619 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -19,133 +19,60 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder;
 import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.cube.inmemcubing.InputConverterUnit;
+import org.apache.kylin.cube.inmemcubing.InputConverterUnitForRawData;
 import org.apache.kylin.engine.mr.ByteArrayWritable;
-import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
-import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.IMRInput;
 import org.apache.kylin.engine.mr.MRUtil;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Maps;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
-/**
- */
-public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArrayWritable, ByteArrayWritable> {
-
-    private static final Logger logger = LoggerFactory.getLogger(InMemCuboidMapper.class);
+public class InMemCuboidMapper<KEYIN>
+        extends InMemCuboidMapperBase<KEYIN, Object, ByteArrayWritable, ByteArrayWritable, String[]> {
 
-    private CubeInstance cube;
-    private CubeDesc cubeDesc;
-    private CubeSegment cubeSegment;
-    private IMRTableInputFormat flatTableInputFormat;
 
-    private BlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(64);
-    private Future<?> future;
+    private IMRInput.IMRTableInputFormat flatTableInputFormat;
 
     @Override
     protected void setup(Context context) throws IOException {
-        super.bindCurrentConfiguration(context.getConfiguration());
+        super.setup(context);
 
-        Configuration conf = context.getConfiguration();
-
-        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
-        String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
-        cube = CubeManager.getInstance(config).getCube(cubeName);
-        cubeDesc = cube.getDescriptor();
-        String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
-        cubeSegment = cube.getSegmentById(segmentID);
         flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSegment).getFlatTableInputFormat();
-        IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
-
-        Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
-
-        // dictionary
-        for (TblColRef col : cubeDesc.getAllColumnsHaveDictionary()) {
-            Dictionary<?> dict = cubeSegment.getDictionary(col);
-            if (dict == null) {
-                logger.warn("Dictionary for " + col + " was not found.");
-            }
-
-            dictionaryMap.put(col, cubeSegment.getDictionary(col));
-        }
-
-        int taskCount = config.getCubeAlgorithmInMemConcurrentThreads();
-        DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cubeSegment.getCuboidScheduler(), flatDesc,
-                dictionaryMap);
-        cubeBuilder.setReserveMemoryMB(calculateReserveMB(context.getConfiguration()));
-        cubeBuilder.setConcurrentThreads(taskCount);
-
-        ExecutorService executorService = Executors.newSingleThreadExecutor();
-        future = executorService.submit(cubeBuilder.buildAsRunnable(queue, new MapContextGTRecordWriter(context, cubeDesc, cubeSegment)));
-
-    }
-
-    private int calculateReserveMB(Configuration configuration) {
-        int sysAvailMB = MemoryBudgetController.getSystemAvailMB();
-        int mrReserve = configuration.getInt("mapreduce.task.io.sort.mb", 100);
-        int sysReserve = Math.max(sysAvailMB / 10, 100);
-        int reserveMB = mrReserve + sysReserve;
-        logger.info("Reserve " + reserveMB + " MB = " + mrReserve + " (MR reserve) + " + sysReserve + " (SYS reserve)");
-        return reserveMB;
     }
 
     @Override
-    public void doMap(KEYIN key, Object record, Context context) throws IOException, InterruptedException {
-        // put each row to the queue
-        Collection<String[]> rowCollection = flatTableInputFormat.parseMapperInput(record);
+    protected InputConverterUnit<String[]> getInputConverterUnit() {
+        Preconditions.checkNotNull(cubeDesc);
+        Preconditions.checkNotNull(dictionaryMap);
+        return new InputConverterUnitForRawData(cubeDesc, flatDesc, dictionaryMap);
 
-        for(String[] row: rowCollection) {
-            List<String> rowAsList = Arrays.asList(row);
-            while (!future.isDone()) {
-                if (queue.offer(rowAsList, 1, TimeUnit.SECONDS)) {
-                    break;
-                }
-            }
-        }
     }
 
     @Override
-    protected void doCleanup(Context context) throws IOException, InterruptedException {
-        logger.info("Totally handled " + mapCounter + " records!");
-
-        while (!future.isDone()) {
-            if (queue.offer(Collections.<String> emptyList(), 1, TimeUnit.SECONDS)) {
-                break;
-            }
-        }
-
-        try {
-            future.get();
-        } catch (Exception e) {
-            throw new IOException("Failed to build cube in mapper " + context.getTaskAttemptID().getTaskID().getId(), e);
-        }
-        queue.clear();
+    protected String[] getRecordFromKeyValue(KEYIN key, Object value) {
+        return flatTableInputFormat.parseMapperInput(value).iterator().next();
     }
 
+    @Override
+    protected Future getCubingThreadFuture(Context context, Map<TblColRef, Dictionary<String>> dictionaryMap,
+            int reserveMemoryMB, CuboidScheduler cuboidScheduler, InputConverterUnit<String[]> inputConverterUnit) {
+        AbstractInMemCubeBuilder cubeBuilder = new DoggedCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap);
+        cubeBuilder.setReserveMemoryMB(reserveMemoryMB);
+
+        ExecutorService executorService = Executors.newSingleThreadExecutor(
+                new ThreadFactoryBuilder().setDaemon(true).setNameFormat("inmemory-cube-building-mapper-%d").build());
+        return executorService.submit(cubeBuilder.buildAsRunnable(queue, inputConverterUnit,
+                new MapContextGTRecordWriter(context, cubeDesc, cubeSegment)));
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
new file mode 100644
index 0000000..0642552
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.MemoryBudgetController;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
+import org.apache.kylin.cube.cuboid.TreeCuboidSchedulerManager;
+import org.apache.kylin.cube.inmemcubing.ConsumeBlockingQueueController;
+import org.apache.kylin.cube.inmemcubing.InputConverterUnit;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+
+/**
+ */
+public abstract class InMemCuboidMapperBase<KEYIN, VALUEIN, KEYOUT, VALUEOUT, T> extends KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
+
+    private static final Logger logger = LoggerFactory.getLogger(InMemCuboidMapperBase.class);
+
+    private int reserveMemoryMB;
+    private int nSplit = 1;
+    private int countOfLastSplit = 0;
+    private int counter = 0;
+    private int splitRowThreshold = Integer.MAX_VALUE;
+    private int unitRows = ConsumeBlockingQueueController.DEFAULT_BATCH_SIZE;
+
+    protected CubeInstance cube;
+    protected CubeDesc cubeDesc;
+    protected CubeSegment cubeSegment;
+    protected Map<TblColRef, Dictionary<String>> dictionaryMap;
+    protected IJoinedFlatTableDesc flatDesc;
+
+    protected BlockingQueue<T> queue = new LinkedBlockingQueue<>(2000);
+    protected InputConverterUnit<T> inputConverterUnit;
+    private Future<?> future;
+
+    protected abstract InputConverterUnit<T> getInputConverterUnit();
+
+    protected abstract Future getCubingThreadFuture(Context context, Map<TblColRef, Dictionary<String>> dictionaryMap, int reserveMemoryMB, //
+                                                    CuboidScheduler cuboidScheduler, InputConverterUnit<T> inputConverterUnit);
+
+    protected abstract T getRecordFromKeyValue(KEYIN key, VALUEIN value);
+
+    @Override
+    protected void setup(Context context) throws IOException {
+        super.bindCurrentConfiguration(context.getConfiguration());
+
+        Configuration conf = context.getConfiguration();
+
+        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+        String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+        cube = CubeManager.getInstance(config).getCube(cubeName);
+        cubeDesc = cube.getDescriptor();
+        String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
+        cubeSegment = cube.getSegmentById(segmentID);
+        flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
+
+        dictionaryMap = Maps.newHashMap();
+
+        // dictionary
+        for (TblColRef col : cubeDesc.getAllColumnsHaveDictionary()) {
+            Dictionary<?> dict = cubeSegment.getDictionary(col);
+            if (dict == null) {
+                logger.warn("Dictionary for " + col + " was not found.");
+            }
+
+            dictionaryMap.put(col, cubeSegment.getDictionary(col));
+        }
+
+        // check memory more often if a single row is big
+        if (cubeDesc.hasMemoryHungryMeasures()) {
+            unitRows /= 10;
+        }
+
+        String cuboidModeName = conf.get(BatchConstants.CFG_CUBOID_MODE);
+        CuboidScheduler cuboidScheduler = TreeCuboidSchedulerManager.getTreeCuboidScheduler(cubeDesc, //
+                CuboidStatsReaderUtil.readCuboidStatsFromSegment(cube.getCuboidsByMode(cuboidModeName), cubeSegment));
+        if (cuboidScheduler == null) {
+            cuboidScheduler = new DefaultCuboidScheduler(cubeDesc);
+        }
+
+        reserveMemoryMB = calculateReserveMB(conf);
+        inputConverterUnit = getInputConverterUnit();
+        future = getCubingThreadFuture(context, dictionaryMap, reserveMemoryMB, cuboidScheduler, inputConverterUnit);
+    }
+
+    private int calculateReserveMB(Configuration configuration) {
+        int sysAvailMB = MemoryBudgetController.getSystemAvailMB();
+        int mrReserve = configuration.getInt("mapreduce.task.io.sort.mb", 100);
+        int sysReserve = Math.max(sysAvailMB / 10, 100);
+        int reserveMB = mrReserve + sysReserve;
+        logger.info("Reserve " + reserveMB + " MB = " + mrReserve + " (MR reserve) + " + sysReserve + " (SYS reserve)");
+        return reserveMB;
+    }
+
+    @Override
+    public void doMap(KEYIN key, VALUEIN value, Context context) throws IOException, InterruptedException {
+        // put each row to the queue
+        T row = getRecordFromKeyValue(key, value);
+
+        if (offer(context, row, 1, TimeUnit.MINUTES, 60)) {
+            counter++;
+            countOfLastSplit++;
+            if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+                logger.info("Handled " + counter + " records, internal queue size = " + queue.size());
+            }
+        } else {
+            throw new IOException("Failed to offer row to internal queue due to queue full!");
+        }
+
+        if (counter % unitRows == 0 && shouldCutSplit(nSplit, countOfLastSplit)) {
+            if (offer(context, inputConverterUnit.getCutUnit(), 1, TimeUnit.MINUTES, 60)) {
+                countOfLastSplit = 0;
+            } else {
+                throw new IOException("Failed to offer row to internal queue due to queue full!");
+            }
+            nSplit++;
+        }
+    }
+
+    @Override
+    protected void doCleanup(Context context) throws IOException, InterruptedException {
+        logger.info("Totally handled " + mapCounter + " records!");
+
+        while (!future.isDone()) {
+            if (queue.offer(inputConverterUnit.getEmptyUnit(), 1, TimeUnit.SECONDS)) {
+                break;
+            }
+        }
+
+        futureGet(context);
+        queue.clear();
+    }
+
+    private boolean shouldCutSplit(int nSplit, long splitRowCount) {
+        int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
+
+        logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold");
+
+        if (splitRowCount >= splitRowThreshold) {
+            logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold);
+            return true;
+        }
+
+        if (systemAvailMB <= reserveMemoryMB) {
+            logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + " MB");
+            return true;
+        }
+
+        return false;
+    }
+
+    private boolean offer(Context context, T row, long timeout, TimeUnit unit, int nRound) throws IOException, InterruptedException {
+        while (nRound > 0) {
+            if (queue.offer(row, timeout, unit)) {
+                return true;
+            }
+            if (future.isDone()) {
+                futureGet(context);
+                throw new IOException("Failed to build cube in mapper due to cubing thread exit unexpectedly");
+            }
+            nRound--;
+        }
+        return false;
+    }
+
+    private void futureGet(Context context) throws IOException {
+        try {
+            future.get();
+        } catch (Exception e) {
+            throw new IOException("Failed to build cube in mapper " + context.getTaskAttemptID().getTaskID().getId(), e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index fd53c5a..d7bc172 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -21,8 +21,6 @@ import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -62,6 +60,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder;
 import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
+import org.apache.kylin.cube.inmemcubing.InputConverterUnitForRawData;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
@@ -249,7 +248,8 @@ public class SparkCubing extends AbstractApplication {
         }
     }
 
-    private Map<Long, HLLCounter> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception {
+    private Map<Long, HLLCounter> sampling(final JavaRDD<String[]> rowJavaRDD, final String cubeName, String segmentId)
+            throws Exception {
         CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
         CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
         CubeDesc cubeDesc = cubeInstance.getDescriptor();
@@ -286,52 +286,55 @@ public class SparkCubing extends AbstractApplication {
             row_hashcodes[i] = new ByteArray();
         }
 
-        final HashMap<Long, HLLCounter> samplingResult = rowJavaRDD.aggregate(zeroValue, new Function2<HashMap<Long, HLLCounter>, List<String>, HashMap<Long, HLLCounter>>() {
+        final HashMap<Long, HLLCounter> samplingResult = rowJavaRDD.aggregate(zeroValue,
+                new Function2<HashMap<Long, HLLCounter>, String[], HashMap<Long, HLLCounter>>() {
 
-            final HashFunction hashFunction = Hashing.murmur3_128();
+                    final HashFunction hashFunction = Hashing.murmur3_128();
 
-            @Override
-            public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, List<String> v2) throws Exception {
-                for (int i = 0; i < nRowKey; i++) {
-                    Hasher hc = hashFunction.newHasher();
-                    String colValue = v2.get(rowKeyColumnIndexes[i]);
-                    if (colValue != null) {
-                        row_hashcodes[i].set(hc.putString(colValue).hash().asBytes());
-                    } else {
-                        row_hashcodes[i].set(hc.putInt(0).hash().asBytes());
-                    }
-                }
+                    @Override
+                    public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, String[] v2) throws Exception {
+                        for (int i = 0; i < nRowKey; i++) {
+                            Hasher hc = hashFunction.newHasher();
+                            String colValue = v2[rowKeyColumnIndexes[i]];
+                            if (colValue != null) {
+                                row_hashcodes[i].set(hc.putString(colValue).hash().asBytes());
+                            } else {
+                                row_hashcodes[i].set(hc.putInt(0).hash().asBytes());
+                            }
+                        }
 
-                for (Map.Entry<Long, Integer[]> entry : allCuboidsBitSet.entrySet()) {
-                    Hasher hc = hashFunction.newHasher();
-                    HLLCounter counter = v1.get(entry.getKey());
-                    final Integer[] cuboidBitSet = entry.getValue();
-                    for (int position = 0; position < cuboidBitSet.length; position++) {
-                        hc.putBytes(row_hashcodes[cuboidBitSet[position]].array());
+                        for (Map.Entry<Long, Integer[]> entry : allCuboidsBitSet.entrySet()) {
+                            Hasher hc = hashFunction.newHasher();
+                            HLLCounter counter = v1.get(entry.getKey());
+                            final Integer[] cuboidBitSet = entry.getValue();
+                            for (int position = 0; position < cuboidBitSet.length; position++) {
+                                hc.putBytes(row_hashcodes[cuboidBitSet[position]].array());
+                            }
+                            counter.add(hc.hash().asBytes());
+                        }
+                        return v1;
+                    }
+                }, new Function2<HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>>() {
+                    @Override
+                    public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, HashMap<Long, HLLCounter> v2)
+                            throws Exception {
+                        Preconditions.checkArgument(v1.size() == v2.size());
+                        Preconditions.checkArgument(v1.size() > 0);
+                        for (Map.Entry<Long, HLLCounter> entry : v1.entrySet()) {
+                            final HLLCounter counter1 = entry.getValue();
+                            final HLLCounter counter2 = v2.get(entry.getKey());
+                            counter1.merge(Preconditions.checkNotNull(counter2, "counter cannot be null"));
+                        }
+                        return v1;
                     }
-                    counter.add(hc.hash().asBytes());
-                }
-                return v1;
-            }
-        }, new Function2<HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>>() {
-            @Override
-            public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, HashMap<Long, HLLCounter> v2) throws Exception {
-                Preconditions.checkArgument(v1.size() == v2.size());
-                Preconditions.checkArgument(v1.size() > 0);
-                for (Map.Entry<Long, HLLCounter> entry : v1.entrySet()) {
-                    final HLLCounter counter1 = entry.getValue();
-                    final HLLCounter counter2 = v2.get(entry.getKey());
-                    counter1.merge(Preconditions.checkNotNull(counter2, "counter cannot be null"));
-                }
-                return v1;
-            }
 
-        });
+                });
         return samplingResult;
     }
 
     /** return hfile location */
-    private String build(JavaRDD<List<String>> javaRDD, final String cubeName, final String segmentId, final byte[][] splitKeys) throws Exception {
+    private String build(JavaRDD<String[]> javaRDD, final String cubeName, final String segmentId,
+            final byte[][] splitKeys) throws Exception {
         CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
         CubeDesc cubeDesc = cubeInstance.getDescriptor();
         final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
@@ -365,36 +368,41 @@ public class SparkCubing extends AbstractApplication {
             }
         }
 
-        final JavaPairRDD<byte[], byte[]> javaPairRDD = javaRDD.glom().mapPartitionsToPair(new PairFlatMapFunction<Iterator<List<List<String>>>, byte[], byte[]>() {
+        final JavaPairRDD<byte[], byte[]> javaPairRDD = javaRDD.glom()
+                .mapPartitionsToPair(new PairFlatMapFunction<Iterator<List<String[]>>, byte[], byte[]>() {
 
-            @Override
-            public Iterator<Tuple2<byte[], byte[]>> call(Iterator<List<List<String>>> listIterator) throws Exception {
-                long t = System.currentTimeMillis();
-                prepare();
+                    @Override
+                    public Iterator<Tuple2<byte[], byte[]>> call(Iterator<List<String[]>> listIterator)
+                            throws Exception {
+                        long t = System.currentTimeMillis();
+                        prepare();
 
-                final CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
+                        final CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
+                                .getCube(cubeName);
 
-                LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue();
-                System.out.println("load properties finished");
-                IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
+                        LinkedBlockingQueue<String[]> blockingQueue = new LinkedBlockingQueue();
+                        System.out.println("load properties finished");
+                        IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
                         AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(
                                 cubeSegment.getCuboidScheduler(), flatDesc, dictionaryMap);
-                final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap));
-                Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, sparkCuboidWriter));
-                try {
-                    while (listIterator.hasNext()) {
-                        for (List<String> row : listIterator.next()) {
-                            blockingQueue.put(row);
+                        final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(
+                                new DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap));
+                        Executors.newCachedThreadPool()
+                                .submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, sparkCuboidWriter));
+                        try {
+                            while (listIterator.hasNext()) {
+                                for (String[] row : listIterator.next()) {
+                                    blockingQueue.put(row);
+                                }
+                            }
+                            blockingQueue.put(InputConverterUnitForRawData.EMPTY_ROW);
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
                         }
+                        System.out.println("build partition cost: " + (System.currentTimeMillis() - t) + "ms");
+                        return sparkCuboidWriter.getResult().iterator();
                     }
-                    blockingQueue.put(Collections.<String> emptyList());
-                } catch (Exception e) {
-                    throw new RuntimeException(e);
-                }
-                System.out.println("build partition cost: " + (System.currentTimeMillis() - t) + "ms");
-                return sparkCuboidWriter.getResult().iterator();
-            }
-        });
+                });
 
         KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
         Configuration conf = getConfigurationForHFile(cubeSegment.getStorageLocationIdentifier());
@@ -562,22 +570,22 @@ public class SparkCubing extends AbstractApplication {
         setupClasspath(sc, confPath);
         intermediateTable.cache();
         writeDictionary(intermediateTable, cubeName, segmentId);
-        final JavaRDD<List<String>> rowJavaRDD = intermediateTable.javaRDD().map(new org.apache.spark.api.java.function.Function<Row, List<String>>() {
-            @Override
-            public List<String> call(Row v1) throws Exception {
-                ArrayList<String> result = Lists.newArrayListWithExpectedSize(v1.size());
-                for (int i = 0; i < v1.size(); i++) {
-                    final Object o = v1.get(i);
-                    if (o != null) {
-                        result.add(o.toString());
-                    } else {
-                        result.add(null);
+        final JavaRDD<String[]> rowJavaRDD = intermediateTable.javaRDD()
+                .map(new org.apache.spark.api.java.function.Function<Row, String[]>() {
+                    @Override
+                    public String[] call(Row v1) throws Exception {
+                        String[] result = new String[v1.size()];
+                        for (int i = 0; i < v1.size(); i++) {
+                            final Object o = v1.get(i);
+                            if (o != null) {
+                                result[i] = o.toString();
+                            } else {
+                                result[i] = null;
+                            }
+                        }
+                        return result;
                     }
-                }
-                return result;
-
-            }
-        });
+                });
 
         final Map<Long, HLLCounter> samplingResult = sampling(rowJavaRDD, cubeName, segmentId);
         final byte[][] splitKeys = createHTable(cubeName, segmentId, samplingResult);

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
index 5e5b16a..fa2d792 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -19,7 +19,6 @@
 package org.apache.kylin.cube.inmemcubing;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
@@ -79,7 +78,7 @@ public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
     @Test
     public void test() throws Exception {
 
-        ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000);
+        ArrayBlockingQueue<String[]> queue = new ArrayBlockingQueue<String[]>(1000);
         ExecutorService executorService = Executors.newSingleThreadExecutor();
         long randSeed = System.currentTimeMillis();
 


[08/22] kylin git commit: APACHE-KYLIN-2783: Refactor CuboidScheduler to be extensible

Posted by nj...@apache.org.
APACHE-KYLIN-2783: Refactor CuboidScheduler to be extensible


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2425bf0c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2425bf0c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2425bf0c

Branch: refs/heads/yaho-cube-planner
Commit: 2425bf0c524ad9f58919e57f7cdae4cdb30fdc7e
Parents: b8f0843
Author: Zhong <nj...@apache.org>
Authored: Thu Aug 17 20:17:26 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:38 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/CubeDescManager.java  |   3 -
 .../org/apache/kylin/cube/CubeInstance.java     |  16 ++
 .../java/org/apache/kylin/cube/CubeManager.java |  17 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   5 +
 .../kylin/cube/common/RowKeySplitter.java       |   4 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java    | 179 ++++++-------------
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  21 +--
 .../kylin/cube/cuboid/CuboidScheduler.java      |  18 ++
 .../cube/cuboid/DefaultCuboidScheduler.java     | 156 +++++++++++++++-
 .../inmemcubing/AbstractInMemCubeBuilder.java   |  17 +-
 .../cube/inmemcubing/DoggedCubeBuilder.java     |   9 +-
 .../cube/inmemcubing/InMemCubeBuilder.java      |  11 +-
 .../org/apache/kylin/cube/kv/RowKeyDecoder.java |   4 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  11 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  35 ++--
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   7 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  26 +--
 .../apache/kylin/cube/cuboid/CuboidTest.java    |  60 ++++---
 .../apache/kylin/cube/kv/RowKeyDecoderTest.java |   2 +-
 .../apache/kylin/cube/kv/RowKeyEncoderTest.java |   6 +-
 .../gtrecord/GTCubeStorageQueryBase.java        |   6 +-
 .../kylin/storage/translate/HBaseKeyRange.java  |   2 +-
 .../kylin/engine/mr/BatchCubingJobBuilder.java  |   2 +-
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |   2 +-
 .../engine/mr/common/BaseCuboidBuilder.java     |  14 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   7 +-
 .../mr/steps/FactDistinctColumnsMapper.java     |   2 +-
 .../engine/mr/steps/InMemCuboidMapper.java      |   3 +-
 .../kylin/engine/mr/steps/KVGTRecordWriter.java |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java      |   2 +-
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |   6 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   9 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  10 +-
 .../spark/cube/DefaultTupleConverter.java       |   2 +-
 .../ITDoggedCubeBuilderStressTest.java          |   2 +-
 .../inmemcubing/ITDoggedCubeBuilderTest.java    |   4 +-
 .../inmemcubing/ITInMemCubeBuilderTest.java     |   2 +-
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   4 +-
 .../cube/MeasureTypeOnlyAggrInBaseTest.java     |   4 +-
 40 files changed, 410 insertions(+), 284 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 6635366..f49c9be 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -171,7 +171,6 @@ public class CubeDescManager {
         CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), false);
 
         cubeDescMap.putLocal(ndesc.getName(), ndesc);
-        Cuboid.reloadCache(name);
 
         // if related cube is in DESCBROKEN state before, change it back to DISABLED
         CubeManager cubeManager = CubeManager.getInstance(config);
@@ -297,13 +296,11 @@ public class CubeDescManager {
         String path = cubeDesc.getResourcePath();
         getStore().deleteResource(path);
         cubeDescMap.remove(cubeDesc.getName());
-        Cuboid.reloadCache(cubeDesc.getName());
     }
 
     // remove cubeDesc
     public void removeLocalCubeDesc(String name) throws IOException {
         cubeDescMap.removeLocal(name);
-        Cuboid.reloadCache(name);
     }
 
     private void reloadAllCubeDesc() throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index fb9a7a7..6369b38 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -27,6 +27,7 @@ import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -93,10 +94,25 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
     @JsonProperty("create_time_utc")
     private long createTimeUTC;
 
+    // cuboid scheduler lazy built
+    transient private CuboidScheduler cuboidScheduler;
+
     // default constructor for jackson
     public CubeInstance() {
     }
 
+    public CuboidScheduler getCuboidScheduler() {
+        if (cuboidScheduler != null)
+            return cuboidScheduler;
+
+        synchronized (this) {
+            if (cuboidScheduler == null) {
+                cuboidScheduler = getDescriptor().getInitialCuboidScheduler();
+            }
+        }
+        return cuboidScheduler;
+    }
+
     public List<CubeSegment> getBuildingSegments() {
         return segments.getBuildingSegments();
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4215746..67c1562 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -43,6 +43,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.dict.DictionaryInfo;
@@ -304,13 +305,14 @@ public class CubeManager implements IRealizationProvider {
         // delete cube instance and cube desc
         CubeInstance cube = getCube(cubeName);
 
-        if (deleteDesc && cube.getDescriptor() != null) {
-            CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
-        }
-
         // remove cube and update cache
         getStore().deleteResource(cube.getResourcePath());
         cubeMap.remove(cube.getName());
+        Cuboid.reloadCache(cube);
+
+        if (deleteDesc && cube.getDescriptor() != null) {
+            CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
+        }
 
         // delete cube from project
         ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
@@ -658,17 +660,20 @@ public class CubeManager implements IRealizationProvider {
      * @param cubeName
      */
     public CubeInstance reloadCubeLocal(String cubeName) {
-        return reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
+        CubeInstance cubeInstance = reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
+        Cuboid.reloadCache(cubeInstance);
+        return cubeInstance;
     }
 
     public void removeCubeLocal(String cubeName) {
         CubeInstance cube = cubeMap.get(cubeName);
         if (cube != null) {
+            cubeMap.removeLocal(cubeName);
             for (CubeSegment segment : cube.getSegments()) {
                 usedStorageLocation.remove(segment.getUuid());
             }
+            Cuboid.reloadCache(cube);
         }
-        cubeMap.removeLocal(cubeName);
     }
 
     public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 1b28bd8..5b2a644 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -31,6 +31,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ShardingHash;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -119,6 +120,10 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
         return getCubeInstance().getDescriptor();
     }
 
+    public CuboidScheduler getCuboidScheduler() {
+        return getCubeInstance().getCuboidScheduler();
+    }
+
     /**
      * @param startDate
      * @param endDate

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index acebce4..cd26347 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 
 public class RowKeySplitter implements java.io.Serializable {
 
+    private CubeSegment cubeSegment;
     private CubeDesc cubeDesc;
     private RowKeyColumnIO colIO;
 
@@ -63,6 +64,7 @@ public class RowKeySplitter implements java.io.Serializable {
     }
 
     public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
+        this.cubeSegment = cubeSeg;
         this.enableSharding = cubeSeg.isEnableSharding();
         this.cubeDesc = cubeSeg.getCubeDesc();
         IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);
@@ -113,7 +115,7 @@ public class RowKeySplitter implements java.io.Serializable {
         offset += RowConstants.ROWKEY_CUBOIDID_LEN;
 
         long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
-        Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId);
+        Cuboid cuboid = Cuboid.findById(cubeSegment, lastSplittedCuboidId);
 
         // rowkey columns
         for (int i = 0; i < cuboid.getColumns().size(); i++) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index b71608c..b2cf339 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -21,15 +21,15 @@ package org.apache.kylin.cube.cuboid;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
@@ -38,16 +38,15 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Collections2;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 @SuppressWarnings("serial")
 public class Cuboid implements Comparable<Cuboid>, Serializable {
 
-    private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
+    //TODO Guava cache may be better
+    private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = Maps.newConcurrentMap();
 
     // smaller is better
     public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() {
@@ -57,10 +56,22 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         }
     };
 
-    // this is the only entry point for query to find the right cuboid
-    public static Cuboid identifyCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
-        long cuboidID = identifyCuboidId(cubeDesc, dimensions, metrics);
-        return Cuboid.findById(cubeDesc, cuboidID);
+    // this is the only entry point for query to find the right cuboid for a segment
+    public static Cuboid identifyCuboid(CubeSegment cubeSegment, Set<TblColRef> dimensions,
+            Collection<FunctionDesc> metrics) {
+        return identifyCuboid(cubeSegment.getCuboidScheduler(), dimensions, metrics);
+    }
+
+    // this is the only entry point for query to find the right cuboid for a cube instance
+    public static Cuboid identifyCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensions,
+            Collection<FunctionDesc> metrics) {
+        return identifyCuboid(cubeInstance.getCuboidScheduler(), dimensions, metrics);
+    }
+
+    public static Cuboid identifyCuboid(CuboidScheduler cuboidScheduler, Set<TblColRef> dimensions,
+            Collection<FunctionDesc> metrics) {
+        long cuboidID = identifyCuboidId(cuboidScheduler.getCubeDesc(), dimensions, metrics);
+        return Cuboid.findById(cuboidScheduler, cuboidID);
     }
 
     public static long identifyCuboidId(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
@@ -77,32 +88,45 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         return cuboidID;
     }
 
-    // for full cube, no need to translate cuboid
-    public static Cuboid findForFullCube(CubeDesc cube, long cuboidID) {
+    // for mandatory cuboid, no need to translate cuboid
+    public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) {
         return new Cuboid(cube, cuboidID, cuboidID);
     }
 
-    public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
-        return findById(cube, Bytes.toLong(cuboidID));
+    public static Cuboid findById(CuboidScheduler cuboidScheduler, byte[] cuboidID) {
+        return findById(cuboidScheduler, Bytes.toLong(cuboidID));
+    }
+
+    public static Cuboid findById(CubeSegment cubeSegment, long cuboidID) {
+        return findById(cubeSegment.getCuboidScheduler(), cuboidID);
     }
 
-    public static Cuboid findById(CubeDesc cube, long cuboidID) {
-        Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cube.getName());
+    public static Cuboid findById(CubeInstance cubeInstance, long cuboidID) {
+        return findById(cubeInstance.getCuboidScheduler(), cuboidID);
+    }
+
+    @VisibleForTesting
+    static Cuboid findById(CubeDesc cubeDesc, long cuboidID) {
+        return findById(cubeDesc.getInitialCuboidScheduler(), cuboidID);
+    }
+
+    public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) {
+        Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getResponsibleKey());
         if (cubeCache == null) {
-            cubeCache = new ConcurrentHashMap<Long, Cuboid>();
-            CUBOID_CACHE.put(cube.getName(), cubeCache);
+            cubeCache = Maps.newConcurrentMap();
+            CUBOID_CACHE.put(cuboidScheduler.getResponsibleKey(), cubeCache);
         }
         Cuboid cuboid = cubeCache.get(cuboidID);
         if (cuboid == null) {
-            long validCuboidID = translateToValidCuboid(cube, cuboidID);
-            cuboid = new Cuboid(cube, cuboidID, validCuboidID);
+            long validCuboidID = translateToValidCuboid(cuboidScheduler, cuboidID);
+            cuboid = new Cuboid(cuboidScheduler.getCubeDesc(), cuboidID, validCuboidID);
             cubeCache.put(cuboidID, cuboid);
         }
         return cuboid;
     }
 
-    public static boolean isValid(CubeDesc cube, long cuboidID) {
-        return cube.getAllCuboids().contains(cuboidID);
+    public static boolean isValid(CuboidScheduler cuboidScheduler, long cuboidID) {
+        return cuboidScheduler.getAllCuboidIds().contains(cuboidID);
     }
 
     public static long getBaseCuboidId(CubeDesc cube) {
@@ -110,107 +134,14 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
     }
 
     public static Cuboid getBaseCuboid(CubeDesc cube) {
-        return findById(cube, getBaseCuboidId(cube));
+        return findForMandatory(cube, getBaseCuboidId(cube));
     }
 
-    static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
-        long baseCuboidId = getBaseCuboidId(cubeDesc);
-        if (cubeDesc.getAllCuboids().contains(cuboidID)) {
+    static long translateToValidCuboid(CuboidScheduler cuboidScheduler, long cuboidID) {
+        if (isValid(cuboidScheduler, cuboidID)) {
             return cuboidID;
         }
-        List<Long> onTreeCandidates = Lists.newArrayList();
-        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
-            Long candidate = translateToOnTreeCuboid(agg, cuboidID);
-            if (candidate != null) {
-                onTreeCandidates.add(candidate);
-            }
-        }
-
-        if (onTreeCandidates.size() == 0) {
-            return baseCuboidId;
-        }
-
-        long onTreeCandi = Collections.min(onTreeCandidates, cuboidSelectComparator);
-        if (isValid(cubeDesc, onTreeCandi)) {
-            return onTreeCandi;
-        }
-
-        return cubeDesc.getCuboidScheduler().findBestMatchCuboid(onTreeCandi);
-    }
-
-    private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
-        if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
-            //the partial cube might not contain all required dims
-            return null;
-        }
-
-        // add mandantory
-        cuboidID = cuboidID | agg.getMandatoryColumnMask();
-
-        // add hierarchy
-        for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
-            long fullMask = hierarchyMask.fullMask;
-            long intersect = cuboidID & fullMask;
-            if (intersect != 0 && intersect != fullMask) {
-
-                boolean startToFill = false;
-                for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
-                    if (startToFill) {
-                        cuboidID |= hierarchyMask.dims[i];
-                    } else {
-                        if ((cuboidID & hierarchyMask.dims[i]) != 0) {
-                            startToFill = true;
-                            cuboidID |= hierarchyMask.dims[i];
-                        }
-                    }
-                }
-            }
-        }
-
-        // add joint dims
-        for (Long joint : agg.getJoints()) {
-            if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
-                cuboidID = cuboidID | joint;
-            }
-        }
-
-        if (!agg.isOnTree(cuboidID)) {
-            // no column, add one column
-            long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()), agg.getJoints());
-            if (nonJointDims != 0) {
-                long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
-                    @Override
-                    public Long apply(HierarchyMask input) {
-                        return input.fullMask;
-                    }
-                }));
-                if (nonJointNonHierarchy != 0) {
-                    //there exists dim that does not belong to any joint or any hierarchy, that's perfect
-                    return cuboidID | Long.lowestOneBit(nonJointNonHierarchy);
-                } else {
-                    //choose from a hierarchy that does not intersect with any joint dim, only check level 1 
-                    long allJointDims = agg.getJointDimsMask();
-                    for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
-                        long dim = hierarchyMask.allMasks[0];
-                        if ((dim & allJointDims) == 0) {
-                            return cuboidID | dim;
-                        }
-                    }
-                }
-            }
-
-            cuboidID = cuboidID | Collections.min(agg.getJoints(), cuboidSelectComparator);
-            Preconditions.checkState(agg.isOnTree(cuboidID));
-        }
-        return cuboidID;
-    }
-
-    private static long removeBits(long original, Collection<Long> toRemove) {
-        long ret = original;
-        for (Long joint : toRemove) {
-            ret = ret & ~joint;
-        }
-        return ret;
+        return cuboidScheduler.findBestMatchCuboid(cuboidID);
     }
 
     // ============================================================================
@@ -310,8 +241,12 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         CUBOID_CACHE.clear();
     }
 
-    public static void reloadCache(String cubeDescName) {
-        CUBOID_CACHE.remove(cubeDescName);
+    public static void reloadCache(CubeInstance cubeInstance) {
+        reloadCache(cubeInstance.getCuboidScheduler());
+    }
+
+    private static void reloadCache(CuboidScheduler cuboidScheduler) {
+        CUBOID_CACHE.remove(cuboidScheduler.getResponsibleKey());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index d657a43..d27ca7a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -44,7 +44,7 @@ public class CuboidCLI {
     }
 
     public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
-        CuboidScheduler scheduler = cubeDesc.getCuboidScheduler();
+        CuboidScheduler scheduler = cubeDesc.getInitialCuboidScheduler();
         long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
         Collection<Long> cuboidSet = new TreeSet<Long>();
         cuboidSet.add(baseCuboid);
@@ -88,28 +88,29 @@ public class CuboidCLI {
                 //check all valid and invalid
                 for (long i = 0; i < baseCuboid; ++i) {
                     if (cuboidSet.contains(i)) {
-                        if (!Cuboid.isValid(cubeDesc, i)) {
+                        if (!Cuboid.isValid(cubeDesc.getInitialCuboidScheduler(), i)) {
                             throw new RuntimeException();
                         }
 
-                        if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) {
+                        if (Cuboid.translateToValidCuboid(cubeDesc.getInitialCuboidScheduler(), i) != i) {
                             throw new RuntimeException();
                         }
                     } else {
-                        if (Cuboid.isValid(cubeDesc, i)) {
+                        if (Cuboid.isValid(cubeDesc.getInitialCuboidScheduler(), i)) {
                             throw new RuntimeException();
                         }
 
-                        long corrected = Cuboid.translateToValidCuboid(cubeDesc, i);
+                        long corrected = Cuboid.translateToValidCuboid(cubeDesc.getInitialCuboidScheduler(), i);
                         if (corrected == i) {
                             throw new RuntimeException();
                         }
 
-                        if (!Cuboid.isValid(cubeDesc, corrected)) {
+                        if (!Cuboid.isValid(cubeDesc.getInitialCuboidScheduler(), corrected)) {
                             throw new RuntimeException();
                         }
 
-                        if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) {
+                        if (Cuboid.translateToValidCuboid(cubeDesc.getInitialCuboidScheduler(),
+                                corrected) != corrected) {
                             throw new RuntimeException();
                         }
                     }
@@ -125,7 +126,7 @@ public class CuboidCLI {
         long baseCuboid = Cuboid.getBaseCuboidId(cube);
         TreeSet<Long> expectedCuboids = new TreeSet<Long>();
         for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
-            if (Cuboid.isValid(cube, cuboid)) {
+            if (Cuboid.isValid(cube.getInitialCuboidScheduler(), cuboid)) {
                 expectedCuboids.add(cuboid);
             }
         }
@@ -133,10 +134,10 @@ public class CuboidCLI {
     }
 
     public static int[] calculateAllLevelCount(CubeDesc cube) {
-        int levels = cube.getBuildLevel();
+        int levels = cube.getInitialBuildLevel();
         int[] allLevelCounts = new int[levels + 1];
 
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
         LinkedList<Long> nextQueue = new LinkedList<Long>();
         LinkedList<Long> currentQueue = new LinkedList<Long>();
         long baseCuboid = Cuboid.getBaseCuboidId(cube);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 1d8f589..c08b9db 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -63,10 +63,21 @@ abstract public class CuboidScheduler {
     /** Returns a cuboid on the tree that best matches the request cuboid. */
     abstract public long findBestMatchCuboid(long requestCuboid);
 
+    /** Returns the key for what this cuboid scheduler responsible for*/
+    abstract public String getResponsibleKey();
+
     // ============================================================================
     
     private transient List<List<Long>> cuboidsByLayer;
 
+    public long getBaseCuboidId() {
+        return Cuboid.getBaseCuboidId(cubeDesc);
+    }
+
+    public CubeDesc getCubeDesc() {
+        return cubeDesc;
+    }
+
     /**
      * Get cuboids by layer. It's built from pre-expanding tree.
      * @return layered cuboids
@@ -101,4 +112,11 @@ abstract public class CuboidScheduler {
         return cuboidsByLayer;
     }
 
+    /**
+     * Get cuboid level count except base cuboid
+     * @return
+     */
+    public int getBuildLevel() {
+        return getCuboidsByLayer().size() - 1;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index b75acd5..c756079 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -35,9 +35,13 @@ import javax.annotation.Nullable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
 import org.apache.kylin.cube.model.CubeDesc;
 
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -159,26 +163,160 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     }
 
     /**
-     * Get the parent cuboid really on the spanning tree.
-     * @param child an on-tree cuboid
+     * Get the parent cuboid rely on the spanning tree.
+     * @param cuboid an on-tree cuboid
      * @return
      */
-    @Override
-    public long findBestMatchCuboid(long child) {
-        long parent = getOnTreeParent(child);
+    public long findBestMatchCuboid(long cuboid) {
+        return findBestMatchCuboid1(cuboid);
+    }
+
+    public long findBestMatchCuboid2(long cuboid) {
+        long bestParent = doFindBestMatchCuboid2(cuboid, Cuboid.getBaseCuboidId(cubeDesc));
+        if (bestParent < -1) {
+            throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboid);
+        }
+        return bestParent;
+    }
+
+    private long doFindBestMatchCuboid2(long cuboid, long parent) {
+        if (!canDerive(cuboid, parent)) {
+            return -1;
+        }
+        List<Long> children = parent2child.get(parent);
+        List<Long> candidates = Lists.newArrayList();
+        if (children != null) {
+            for (long child : children) {
+                long candidate = doFindBestMatchCuboid2(cuboid, child);
+                if (candidate > 0) {
+                    candidates.add(candidate);
+                }
+            }
+        }
+        if (candidates.isEmpty()) {
+            candidates.add(parent);
+        }
+
+        return Collections.min(candidates, Cuboid.cuboidSelectComparator);
+    }
+
+    private boolean canDerive(long cuboidId, long parentCuboid) {
+        return (cuboidId & ~parentCuboid) == 0;
+    }
+
+    public long findBestMatchCuboid1(long cuboid) {
+        List<Long> onTreeCandidates = Lists.newArrayList();
+        for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+            Long candidate = translateToOnTreeCuboid(agg, cuboid);
+            if (candidate != null) {
+                onTreeCandidates.add(candidate);
+            }
+        }
+
+        if (onTreeCandidates.size() == 0) {
+            return Cuboid.getBaseCuboidId(cubeDesc);
+        }
+
+        long onTreeCandi = Collections.min(onTreeCandidates, Cuboid.cuboidSelectComparator);
+        if (allCuboidIds.contains(onTreeCandi)) {
+            return onTreeCandi;
+        }
+
+        return doFindBestMatchCuboid1(onTreeCandi);
+    }
+
+    public long doFindBestMatchCuboid1(long cuboid) {
+        long parent = getOnTreeParent(cuboid);
         while (parent > 0) {
-            if (cubeDesc.getAllCuboids().contains(parent)) {
+            if (allCuboidIds.contains(parent)) {
                 break;
             }
             parent = getOnTreeParent(parent);
         }
 
         if (parent <= 0) {
-            throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
+            throw new IllegalStateException("Can't find valid parent for Cuboid " + cuboid);
         }
         return parent;
     }
 
+    private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
+        if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
+            //the partial cube might not contain all required dims
+            return null;
+        }
+
+        // add mandantory
+        cuboidID = cuboidID | agg.getMandatoryColumnMask();
+
+        // add hierarchy
+        for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+            long fullMask = hierarchyMask.fullMask;
+            long intersect = cuboidID & fullMask;
+            if (intersect != 0 && intersect != fullMask) {
+
+                boolean startToFill = false;
+                for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
+                    if (startToFill) {
+                        cuboidID |= hierarchyMask.dims[i];
+                    } else {
+                        if ((cuboidID & hierarchyMask.dims[i]) != 0) {
+                            startToFill = true;
+                            cuboidID |= hierarchyMask.dims[i];
+                        }
+                    }
+                }
+            }
+        }
+
+        // add joint dims
+        for (Long joint : agg.getJoints()) {
+            if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
+                cuboidID = cuboidID | joint;
+            }
+        }
+
+        if (!agg.isOnTree(cuboidID)) {
+            // no column, add one column
+            long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()),
+                    agg.getJoints());
+            if (nonJointDims != 0) {
+                long nonJointNonHierarchy = removeBits(nonJointDims,
+                        Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
+                            @Override
+                            public Long apply(HierarchyMask input) {
+                                return input.fullMask;
+                            }
+                        }));
+                if (nonJointNonHierarchy != 0) {
+                    //there exists dim that does not belong to any joint or any hierarchy, that's perfect
+                    return cuboidID | Long.lowestOneBit(nonJointNonHierarchy);
+                } else {
+                    //choose from a hierarchy that does not intersect with any joint dim, only check level 1
+                    long allJointDims = agg.getJointDimsMask();
+                    for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+                        long dim = hierarchyMask.allMasks[0];
+                        if ((dim & allJointDims) == 0) {
+                            return cuboidID | dim;
+                        }
+                    }
+                }
+            }
+
+            cuboidID = cuboidID | Collections.min(agg.getJoints(), Cuboid.cuboidSelectComparator);
+            Preconditions.checkState(agg.isOnTree(cuboidID));
+        }
+        return cuboidID;
+    }
+
+    private static long removeBits(long original, Collection<Long> toRemove) {
+        long ret = original;
+        for (Long joint : toRemove) {
+            ret = ret & ~joint;
+        }
+        return ret;
+    }
+
     private long getOnTreeParent(long child) {
         Collection<Long> candidates = getOnTreeParents(child);
         if (candidates == null || candidates.isEmpty()) {
@@ -257,4 +395,8 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
 
         return parentCandidate;
     }
+
+    public String getResponsibleKey() {
+        return CubeDesc.class.getName() + "-" + cubeDesc.getName();
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index c7a4a05..ae38261 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 
 import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
@@ -42,6 +43,7 @@ abstract public class AbstractInMemCubeBuilder {
 
     private static Logger logger = LoggerFactory.getLogger(AbstractInMemCubeBuilder.class);
 
+    final protected CuboidScheduler cuboidScheduler;
     final protected IJoinedFlatTableDesc flatDesc;
     final protected CubeDesc cubeDesc;
     final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
@@ -49,16 +51,23 @@ abstract public class AbstractInMemCubeBuilder {
     protected int taskThreadCount = 1;
     protected int reserveMemoryMB = 100;
 
-    public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        if (flatDesc == null)
+    //    @Deprecated
+    //    public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+    //        this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap);
+    //    }
+
+    protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        if (cuboidScheduler == null)
             throw new NullPointerException();
-        if (cubeDesc == null)
+        if (flatDesc == null)
             throw new NullPointerException();
         if (dictionaryMap == null)
             throw new IllegalArgumentException("dictionary cannot be null");
 
+        this.cuboidScheduler = cuboidScheduler;
         this.flatDesc = flatDesc;
-        this.cubeDesc = cubeDesc;
+        this.cubeDesc = cuboidScheduler.getCubeDesc();
         this.dictionaryMap = dictionaryMap;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index a9211da..dd92a2b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -35,7 +35,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequestBuilder;
 import org.apache.kylin.gridtable.IGTScanner;
@@ -57,8 +57,9 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
     private int splitRowThreshold = Integer.MAX_VALUE;
     private int unitRows = 1000;
 
-    public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        super(cubeDesc, flatDesc, dictionaryMap);
+    public DoggedCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        super(cuboidScheduler, flatDesc, dictionaryMap);
 
         // check memory more often if a single row is big
         if (cubeDesc.hasMemoryHungryMeasures())
@@ -276,7 +277,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
         RuntimeException exception;
 
         public SplitThread() {
-            this.builder = new InMemCubeBuilder(cubeDesc, flatDesc, dictionaryMap);
+            this.builder = new InMemCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap);
             this.builder.setConcurrentThreads(taskThreadCount);
             this.builder.setReserveMemoryMB(reserveMemoryMB);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 93736c9..684c26b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -39,7 +39,6 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.gridtable.CubeGridTable;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
-import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.gridtable.GTAggregateScanner;
 import org.apache.kylin.gridtable.GTBuilder;
 import org.apache.kylin.gridtable.GTInfo;
@@ -71,7 +70,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     private static final double DERIVE_AGGR_CACHE_CONSTANT_FACTOR = 0.1;
     private static final double DERIVE_AGGR_CACHE_VARIABLE_FACTOR = 0.9;
 
-    private final CuboidScheduler cuboidScheduler;
     private final long baseCuboidId;
     private final int totalCuboidCount;
     private final String[] metricsAggrFuncs;
@@ -90,9 +88,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     private Object[] totalSumForSanityCheck;
     private ICuboidCollector resultCollector;
 
-    public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        super(cubeDesc, flatDesc, dictionaryMap);
-        this.cuboidScheduler = cubeDesc.getCuboidScheduler();
+    public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+            Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        super(cuboidScheduler, flatDesc, dictionaryMap);
         this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
         this.totalCuboidCount = cuboidScheduler.getCuboidCount();
 
@@ -109,8 +107,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
-        GTInfo info = CubeGridTable.newGTInfo(
-                Cuboid.findById(cubeDesc, cuboidID),
+        GTInfo info = CubeGridTable.newGTInfo(Cuboid.findById(cuboidScheduler, cuboidID),
                 new CubeDimEncMap(cubeDesc, dictionaryMap)
         );
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index d5948d4..5a1f668 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -36,6 +36,7 @@ import org.apache.kylin.metadata.model.TblColRef;
  */
 public class RowKeyDecoder {
 
+    private final CubeSegment cubeSegment;
     private final CubeDesc cubeDesc;
     private final RowKeyColumnIO colIO;
     private final RowKeySplitter rowKeySplitter;
@@ -44,6 +45,7 @@ public class RowKeyDecoder {
     private List<String> values;
 
     public RowKeyDecoder(CubeSegment cubeSegment) {
+        this.cubeSegment = cubeSegment;
         this.cubeDesc = cubeSegment.getCubeDesc();
         this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
         this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
@@ -73,7 +75,7 @@ public class RowKeyDecoder {
         if (this.cuboid != null && this.cuboid.getId() == cuboidID) {
             return;
         }
-        this.cuboid = Cuboid.findById(cubeDesc, cuboidID);
+        this.cuboid = Cuboid.findById(cubeSegment, cuboidID);
     }
 
     private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index a669fb1..40cda76 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -18,7 +18,11 @@
 
 package org.apache.kylin.cube.kv;
 
-import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ImmutableBitSet;
@@ -28,10 +32,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.TblColRef;
 
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.base.Preconditions;
 
 public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5d8503d..510a7d5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -52,6 +52,7 @@ import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
@@ -188,9 +189,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
     @JsonInclude(JsonInclude.Include.NON_NULL)
     private int parentForward = 3;
 
-    // cuboid scheduler lazy built
-    transient private CuboidScheduler cuboidScheduler = null;
-
     public boolean isEnableSharding() {
         //in the future may extend to other storage that is shard-able
         return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID;
@@ -458,14 +456,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         return true;
     }
 
-    /**
-     * Get cuboid level count except base cuboid
-     * @return
-     */
-    public int getBuildLevel() {
-        return getCuboidScheduler().getCuboidsByLayer().size() - 1;
-    }
-
     @Override
     public int hashCode() {
         int result = 0;
@@ -556,7 +546,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         hostToDerivedMap = Maps.newHashMap();
         extendedColumnToHosts = Maps.newHashMap();
         cuboidBlackSet = Sets.newHashSet();
-        cuboidScheduler = null;
     }
 
     public void init(KylinConfig config) {
@@ -612,16 +601,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         amendAllColumns();
     }
 
-    public CuboidScheduler getCuboidScheduler() {
-        if (cuboidScheduler != null)
-            return cuboidScheduler;
-        
-        synchronized (this) {
-            if (cuboidScheduler == null) {
-                cuboidScheduler = CuboidScheduler.getInstance(this);
-            }
-            return cuboidScheduler;
-        }
+    public CuboidScheduler getInitialCuboidScheduler() {
+        return new DefaultCuboidScheduler(this);
+    }
+
+    /**
+     * Get cuboid level count except base cuboid
+     * @return
+     */
+    public int getInitialBuildLevel() {
+        return getInitialCuboidScheduler().getCuboidsByLayer().size() - 1;
     }
 
     public boolean isBlackedCuboid(long cuboidID) {
@@ -1142,7 +1131,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
     }
 
     public Set<Long> getAllCuboids() {
-        return getCuboidScheduler().getAllCuboidIds();
+        return getInitialCuboidScheduler().getAllCuboidIds();
     }
 
     public int getParentForward() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index e829aeb..b16fce0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.cube.model;
 
+import java.util.List;
+
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
@@ -26,8 +28,6 @@ import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 
-import java.util.List;
-
 /**
  * An enrich of IJoinedFlatTableDesc for cubes
  */
@@ -50,8 +50,7 @@ public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, java.io.
 
     // check what columns from hive tables are required, and index them
     private void parseCubeDesc() {
-        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
 
         // build index for rowkey columns
         List<TblColRef> cuboidColumns = baseCuboid.getColumns();

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 1c3c395..ec4c04a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -57,7 +57,7 @@ public class CubingUtils {
     public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
         final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
         final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
-        final Set<Long> allCuboidIds = cubeDesc.getCuboidScheduler().getAllCuboidIds();
+        final Set<Long> allCuboidIds = cubeDesc.getInitialCuboidScheduler().getAllCuboidIds();
         final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
         final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMap();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 58c0edb..3992ef9 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -115,7 +115,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
             System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
 
             for (long child : spannings) {
-                assertTrue(Cuboid.isValid(cube, child));
+                assertTrue(Cuboid.isValid(scheduler, child));
             }
         }
 
@@ -128,7 +128,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testGetSpanningCuboid2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         // generate 8d
         System.out.println("Spanning for 8D Cuboids");
@@ -156,7 +156,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testGetSpanningCuboid1() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         // generate 7d
         System.out.println("Spanning for 7D Cuboids");
@@ -228,7 +228,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts1() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -241,7 +241,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts2() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -254,7 +254,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts3() {
         CubeDesc cube = getTestKylinCubeWithSeller();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -267,7 +267,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts4() {
         CubeDesc cube = getTestKylinCubeWithSellerLeft();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -280,7 +280,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts5() {
         CubeDesc cube = getStreamingCubeDesc();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -293,7 +293,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testCuboidCounts6() {
         CubeDesc cube = getCIInnerJoinCube();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         int[] counts = CuboidCLI.calculateAllLevelCount(cube);
         printCount(counts);
         int sum = 0;
@@ -306,7 +306,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
     @Test
     public void testLargeCube() {
         CubeDesc cube = getFiftyDimCubeDesc();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         long start = System.currentTimeMillis();
         System.out.println(cuboidScheduler.getCuboidCount());
         System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms");
@@ -317,7 +317,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
         twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4)));
         CubeDesc cube = getTwentyDimCubeDesc();
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         cuboidScheduler.getCuboidCount();
         twentyFile.renameTo(new File(twentyFile.getPath() + ".bad"));
     }
@@ -332,10 +332,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         }
         CubeDescManager.clearCache();
         CubeDesc cube = getCubeDescManager().getCubeDesc("ut_large_dimension_number");
-        CuboidScheduler scheduler = cube.getCuboidScheduler();
+        CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
-        assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+        assertTrue(Cuboid.isValid(scheduler, baseCuboid.getId()));
 
         List<Long> spanningChild = scheduler.getSpanningCuboid(baseCuboid.getId());
         assertTrue(spanningChild.size() > 0);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 2e64791..ab0c666 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -73,25 +73,26 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     public void testIsValid() {
 
         CubeDesc cube = getTestKylinCubeWithSeller();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
         // base
-        assertEquals(false, Cuboid.isValid(cube, 0));
-        assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, 0));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("111111111")));
 
         // mandatory column
-        assertEquals(false, Cuboid.isValid(cube, toLong("011111110")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("100000000")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("011111110")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("100000000")));
 
         // zero tail
-        assertEquals(true, Cuboid.isValid(cube, toLong("111111000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("111111000")));
 
         // aggregation group & zero tail
-        assertEquals(true, Cuboid.isValid(cube, toLong("110000111")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("110111000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("111110111")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("111110001")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("111110100")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("110000100")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("110000111")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("110111000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("111110111")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("111110001")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("111110100")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("110000100")));
     }
 
     @Test
@@ -124,41 +125,45 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testIsValid2() {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
-        assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
+
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("111111111")));
 
         // base
-        assertEquals(false, Cuboid.isValid(cube, 0));
-        assertEquals(true, Cuboid.isValid(cube, toLong("11111111")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, 0));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("11111111")));
 
         // aggregation group & zero tail
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000111")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("10001111")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("11001111")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000001")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000101")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000111")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10001111")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("11001111")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000001")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000101")));
 
         // hierarchy
-        assertEquals(true, Cuboid.isValid(cube, toLong("10100000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10110000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("10111000")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("10001000")));
-        assertEquals(false, Cuboid.isValid(cube, toLong("10011000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10100000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10110000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10111000")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10001000")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10011000")));
     }
 
     @Test
     public void testIsValid3() {
         CubeDesc cube = getSSBCubeDesc();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
-        assertEquals(false, Cuboid.isValid(cube, toLong("10000000000")));
+        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10000000000")));
 
         // the 4th is mandatory and isMandatoryOnlyValid is true
-        assertEquals(true, Cuboid.isValid(cube, toLong("10000001000")));
-        assertEquals(true, Cuboid.isValid(cube, toLong("00000001000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000001000")));
+        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("00000001000")));
     }
 
     @Test
     public void testFindCuboidByIdWithSingleAggrGroup2() {
         CubeDesc cube = getTestKylinCubeWithSeller();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, 0);
@@ -186,6 +191,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
     @Test
     public void testFindCuboidByIdWithMultiAggrGroup() {
         CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
+        CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
         Cuboid cuboid;
 
         cuboid = Cuboid.findById(cube, toLong("111111110"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index 1d1d147..ec1f221 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -91,7 +91,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
         data[7] = "15";
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index 75e2458..5af8d8a 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -66,7 +66,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
         data[7] = "15";
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);
@@ -97,7 +97,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
         data[8] = "15";
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);
@@ -133,7 +133,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
         data[8] = null;
 
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
         RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
 
         byte[] encodedKey = rowKeyEncoder.encode(data);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b95253d..d7a5652 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -129,7 +129,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
         Set<TblColRef> dimensionsD = new LinkedHashSet<TblColRef>();
         dimensionsD.addAll(groupsD);
         dimensionsD.addAll(otherDimsD);
-        Cuboid cuboid = findCuboid(cubeDesc, dimensionsD, metrics);
+        Cuboid cuboid = findCuboid(cubeInstance, dimensionsD, metrics);
         context.setCuboid(cuboid);
 
         // set whether to aggr at storage
@@ -160,8 +160,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
 
     protected abstract String getGTStorage();
 
-    protected Cuboid findCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) {
-        return Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
+    protected Cuboid findCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) {
+        return Cuboid.identifyCuboid(cubeInstance, dimensionsD, metrics);
     }
 
     protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index bfddb1f..f49b728 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -84,7 +84,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
     public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) {
         this.cubeSeg = cubeSeg;
         long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns);
-        this.cuboid = Cuboid.findById(cubeDesc, cuboidId);
+        this.cuboid = Cuboid.findById(cubeSeg, cuboidId);
         this.flatOrAndFilter = Lists.newLinkedList();
         this.flatOrAndFilter.add(andDimensionRanges);
         init(andDimensionRanges);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 1ec23b6..f64365a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -64,7 +64,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
 
         // Phase 3: Build Cube
         RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
-        final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
+        final int groupRowkeyColumnsCount = seg.getCuboidScheduler().getBuildLevel();
         // base cuboid step
         result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
         // n dim cuboid steps

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 106077c..51c9056 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -76,7 +76,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
     }
 
     protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
-        final int maxLevel = seg.getCubeDesc().getBuildLevel();
+        final int maxLevel = seg.getCuboidScheduler().getBuildLevel();
         // base cuboid step
         result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
         // n dim cuboid steps

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
index 07b636b..40f1ac5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -18,7 +18,11 @@
 
 package org.apache.kylin.engine.mr.common;
 
-import com.google.common.collect.Sets;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.cube.CubeSegment;
@@ -35,10 +39,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.collect.Sets;
 
 /**
  */
@@ -88,8 +89,7 @@ public class BaseCuboidBuilder implements java.io.Serializable {
     }
 
     private void init() {
-        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
         initNullBytes();
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index e160d27..417697d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -81,7 +81,7 @@ public class CubeStatsReader {
 
     public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
         ResourceStore store = ResourceStore.getStore(kylinConfig);
-        cuboidScheduler = cubeSegment.getCubeDesc().getCuboidScheduler();
+        cuboidScheduler = cubeSegment.getCuboidScheduler();
         String statsKey = cubeSegment.getStatisticsResourcePath();
         File tmpSeqFile = writeTmpSeqFile(store.getResource(statsKey).inputStream);
         Reader reader = null;
@@ -174,8 +174,7 @@ public class CubeStatsReader {
     public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap) {
         final CubeDesc cubeDesc = cubeSegment.getCubeDesc();
         final List<Integer> rowkeyColumnSize = Lists.newArrayList();
-        final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-        final Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
         final List<TblColRef> columnList = baseCuboid.getColumns();
         final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap();
 
@@ -185,7 +184,7 @@ public class CubeStatsReader {
 
         Map<Long, Double> sizeMap = Maps.newHashMap();
         for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) {
-            sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(), baseCuboidId, rowkeyColumnSize));
+            sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(), baseCuboid.getId(), rowkeyColumnSize));
         }
         return sizeMap;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 8281759..16bc03c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -88,7 +88,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
         collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
         if (collectStatistics) {
             samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
-            cuboidScheduler = cubeDesc.getCuboidScheduler();
+            cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
             nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
 
             List<Long> cuboidIdList = Lists.newArrayList();

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index eee189c..5511414 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -96,7 +96,8 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
         }
 
         int taskCount = config.getCubeAlgorithmInMemConcurrentThreads();
-        DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
+        DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cubeSegment.getCuboidScheduler(), flatDesc,
+                dictionaryMap);
         cubeBuilder.setReserveMemoryMB(calculateReserveMB(context.getConfiguration()));
         cubeBuilder.setConcurrentThreads(taskCount);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
index aa323fd..2058bc9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
@@ -95,7 +95,7 @@ public abstract class KVGTRecordWriter implements ICuboidWriter {
     protected abstract void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException;
 
     private void initVariables(Long cuboidId) {
-        rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
+        rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeSegment, cuboidId));
         keyBuf = rowKeyEncoder.createBuf();
 
         dimensions = Long.bitCount(cuboidId);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index a01a928..c80283e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -150,7 +150,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
     @Override
     public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
         long cuboidID = rowKeySplitter.split(key.getBytes());
-        Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidID);
+        Cuboid cuboid = Cuboid.findById(cube, cuboidID);
         RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid);
 
         SplittedBytes[] splittedByteses = rowKeySplitter.getSplitBuffers();

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 782ce72..6680fd7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -75,7 +75,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
         cubeDesc = cube.getDescriptor();
         ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
         // initialize CubiodScheduler
-        cuboidScheduler = cubeDesc.getCuboidScheduler();
+        cuboidScheduler = cubeSegment.getCuboidScheduler();
         rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
     }
 
@@ -84,7 +84,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
     @Override
     public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
         long cuboidId = rowKeySplitter.split(key.getBytes());
-        Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
+        Cuboid parentCuboid = Cuboid.findById(cuboidScheduler, cuboidId);
 
         Collection<Long> myChildren = cuboidScheduler.getSpanningCuboid(cuboidId);
 
@@ -104,7 +104,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
         }
 
         for (Long child : myChildren) {
-            Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
+            Cuboid childCuboid = Cuboid.findById(cuboidScheduler, child);
             Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
             outputKey.set(result.getSecond().array(), 0, result.getFirst());
             context.write(outputKey, value);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2425bf0c/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index a094cc2..fd53c5a 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -189,7 +189,7 @@ public class SparkCubing extends AbstractApplication {
         final CubeDesc cubeDesc = cubeInstance.getDescriptor();
         final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap();
         final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(seg), cubeDesc);
-        final List<TblColRef> baseCuboidColumn = Cuboid.findById(cubeDesc, Cuboid.getBaseCuboidId(cubeDesc)).getColumns();
+        final List<TblColRef> baseCuboidColumn = Cuboid.findById(seg, Cuboid.getBaseCuboidId(cubeDesc)).getColumns();
         final long start = System.currentTimeMillis();
         final RowKeyDesc rowKey = cubeDesc.getRowkey();
         for (int i = 0; i < baseCuboidColumn.size(); i++) {
@@ -253,7 +253,7 @@ public class SparkCubing extends AbstractApplication {
         CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
         CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
         CubeDesc cubeDesc = cubeInstance.getDescriptor();
-        CuboidScheduler cuboidScheduler = cubeDesc.getCuboidScheduler();
+        CuboidScheduler cuboidScheduler = cubeInstance.getCuboidScheduler();
         Set<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds();
         final HashMap<Long, HLLCounter> zeroValue = Maps.newHashMap();
         for (Long id : allCuboidIds) {
@@ -335,7 +335,7 @@ public class SparkCubing extends AbstractApplication {
         CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
         CubeDesc cubeDesc = cubeInstance.getDescriptor();
         final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
-        List<TblColRef> baseCuboidColumn = Cuboid.findById(cubeDesc, Cuboid.getBaseCuboidId(cubeDesc)).getColumns();
+        List<TblColRef> baseCuboidColumn = Cuboid.findById(cubeSegment, Cuboid.getBaseCuboidId(cubeDesc)).getColumns();
         final Map<TblColRef, Integer> columnLengthMap = Maps.newHashMap();
         final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap();
         for (TblColRef tblColRef : baseCuboidColumn) {
@@ -377,7 +377,8 @@ public class SparkCubing extends AbstractApplication {
                 LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue();
                 System.out.println("load properties finished");
                 IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
-                AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(cubeInstance.getDescriptor(), flatDesc, dictionaryMap);
+                        AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(
+                                cubeSegment.getCuboidScheduler(), flatDesc, dictionaryMap);
                 final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap));
                 Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, sparkCuboidWriter));
                 try {


[05/22] kylin git commit: APACHE-KYLIN-2783: move method isValid from Cuboid to CuboidScheduler

Posted by nj...@apache.org.
APACHE-KYLIN-2783: move method isValid from Cuboid to CuboidScheduler


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6ec7e93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6ec7e93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6ec7e93

Branch: refs/heads/yaho-cube-planner
Commit: b6ec7e93dec0efc01ba67916a928b289d3d67280
Parents: 9b67d58
Author: Zhong <nj...@apache.org>
Authored: Thu Aug 17 21:13:51 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Fri Sep 8 11:31:38 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/cuboid/Cuboid.java    | 13 +----
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 15 +++---
 .../kylin/cube/cuboid/CuboidScheduler.java      |  3 ++
 .../cube/cuboid/DefaultCuboidScheduler.java     | 13 ++++-
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  | 11 +++-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  4 +-
 .../apache/kylin/cube/cuboid/CuboidTest.java    | 54 ++++++++++----------
 7 files changed, 61 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index b2cf339..fa83775 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -118,17 +118,13 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         }
         Cuboid cuboid = cubeCache.get(cuboidID);
         if (cuboid == null) {
-            long validCuboidID = translateToValidCuboid(cuboidScheduler, cuboidID);
+            long validCuboidID = cuboidScheduler.findBestMatchCuboid(cuboidID);
             cuboid = new Cuboid(cuboidScheduler.getCubeDesc(), cuboidID, validCuboidID);
             cubeCache.put(cuboidID, cuboid);
         }
         return cuboid;
     }
 
-    public static boolean isValid(CuboidScheduler cuboidScheduler, long cuboidID) {
-        return cuboidScheduler.getAllCuboidIds().contains(cuboidID);
-    }
-
     public static long getBaseCuboidId(CubeDesc cube) {
         return cube.getRowkey().getFullMask();
     }
@@ -137,13 +133,6 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
         return findForMandatory(cube, getBaseCuboidId(cube));
     }
 
-    static long translateToValidCuboid(CuboidScheduler cuboidScheduler, long cuboidID) {
-        if (isValid(cuboidScheduler, cuboidID)) {
-            return cuboidID;
-        }
-        return cuboidScheduler.findBestMatchCuboid(cuboidID);
-    }
-
     // ============================================================================
 
     private CubeDesc cubeDesc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index d27ca7a..4300a6c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -88,29 +88,28 @@ public class CuboidCLI {
                 //check all valid and invalid
                 for (long i = 0; i < baseCuboid; ++i) {
                     if (cuboidSet.contains(i)) {
-                        if (!Cuboid.isValid(cubeDesc.getInitialCuboidScheduler(), i)) {
+                        if (!cubeDesc.getInitialCuboidScheduler().isValid(i)) {
                             throw new RuntimeException();
                         }
 
-                        if (Cuboid.translateToValidCuboid(cubeDesc.getInitialCuboidScheduler(), i) != i) {
+                        if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i) != i) {
                             throw new RuntimeException();
                         }
                     } else {
-                        if (Cuboid.isValid(cubeDesc.getInitialCuboidScheduler(), i)) {
+                        if (cubeDesc.getInitialCuboidScheduler().isValid(i)) {
                             throw new RuntimeException();
                         }
 
-                        long corrected = Cuboid.translateToValidCuboid(cubeDesc.getInitialCuboidScheduler(), i);
+                        long corrected = cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i);
                         if (corrected == i) {
                             throw new RuntimeException();
                         }
 
-                        if (!Cuboid.isValid(cubeDesc.getInitialCuboidScheduler(), corrected)) {
+                        if (!cubeDesc.getInitialCuboidScheduler().isValid(corrected)) {
                             throw new RuntimeException();
                         }
 
-                        if (Cuboid.translateToValidCuboid(cubeDesc.getInitialCuboidScheduler(),
-                                corrected) != corrected) {
+                        if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(corrected) != corrected) {
                             throw new RuntimeException();
                         }
                     }
@@ -126,7 +125,7 @@ public class CuboidCLI {
         long baseCuboid = Cuboid.getBaseCuboidId(cube);
         TreeSet<Long> expectedCuboids = new TreeSet<Long>();
         for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
-            if (Cuboid.isValid(cube.getInitialCuboidScheduler(), cuboid)) {
+            if (cube.getInitialCuboidScheduler().isValid(cuboid)) {
                 expectedCuboids.add(cuboid);
             }
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index c08b9db..cef0f77 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -63,6 +63,9 @@ abstract public class CuboidScheduler {
     /** Returns a cuboid on the tree that best matches the request cuboid. */
     abstract public long findBestMatchCuboid(long requestCuboid);
 
+    /** Returns whether requestCuboid is valid or not*/
+    abstract public boolean isValid(long requestCuboid);
+
     /** Returns the key for what this cuboid scheduler responsible for*/
     abstract public String getResponsibleKey();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index c756079..d05786e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -87,6 +87,11 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         return Sets.newHashSet(allCuboidIds);
     }
 
+    @Override
+    public boolean isValid(long requestCuboid) {
+        return allCuboidIds.contains(requestCuboid);
+    }
+
     /**
      * Collect cuboid from bottom up, considering all factor including black list
      * Build tree steps:
@@ -205,6 +210,10 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     }
 
     public long findBestMatchCuboid1(long cuboid) {
+        if (isValid(cuboid)) {
+            return cuboid;
+        }
+
         List<Long> onTreeCandidates = Lists.newArrayList();
         for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
             Long candidate = translateToOnTreeCuboid(agg, cuboid);
@@ -218,7 +227,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
         }
 
         long onTreeCandi = Collections.min(onTreeCandidates, Cuboid.cuboidSelectComparator);
-        if (allCuboidIds.contains(onTreeCandi)) {
+        if (isValid(onTreeCandi)) {
             return onTreeCandi;
         }
 
@@ -228,7 +237,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
     public long doFindBestMatchCuboid1(long cuboid) {
         long parent = getOnTreeParent(cuboid);
         while (parent > 0) {
-            if (allCuboidIds.contains(parent)) {
+            if (isValid(parent)) {
                 break;
             }
             parent = getOnTreeParent(parent);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
index 73f4824..414fc4a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -69,6 +69,11 @@ public class TreeCuboidScheduler extends CuboidScheduler {
         return cuboidTree.findBestMatchCuboid(cuboidId);
     }
 
+    @Override
+    public boolean isValid(long requestCuboid) {
+        return cuboidTree.isValid(requestCuboid);
+    }
+
     public static class CuboidTree {
         private int treeLevels;
 
@@ -129,13 +134,17 @@ public class TreeCuboidScheduler extends CuboidScheduler {
 
         public long findBestMatchCuboid(long cuboidId) {
             // exactly match
-            if (index.containsKey(cuboidId)) {
+            if (isValid(cuboidId)) {
                 return cuboidId;
             }
 
             return findBestParent(cuboidId).cuboidId;
         }
 
+        public boolean isValid(long cuboidId) {
+            return index.containsKey(cuboidId);
+        }
+
         private int getCuboidCount(long cuboidId) {
             int r = 1;
             for (Long child : getSpanningCuboid(cuboidId)) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 3992ef9..3f3a7c4 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -115,7 +115,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
             System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
 
             for (long child : spannings) {
-                assertTrue(Cuboid.isValid(scheduler, child));
+                assertTrue(scheduler.isValid(child));
             }
         }
 
@@ -335,7 +335,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
         CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
 
         Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
-        assertTrue(Cuboid.isValid(scheduler, baseCuboid.getId()));
+        assertTrue(scheduler.isValid(baseCuboid.getId()));
 
         List<Long> spanningChild = scheduler.getSpanningCuboid(baseCuboid.getId());
         assertTrue(spanningChild.size() > 0);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6ec7e93/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index ab0c666..8dc944b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -76,23 +76,23 @@ public class CuboidTest extends LocalFileMetadataTestCase {
         CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
         // base
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, 0));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("111111111")));
+        assertEquals(false, cuboidScheduler.isValid(0));
+        assertEquals(true, cuboidScheduler.isValid(toLong("111111111")));
 
         // mandatory column
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("011111110")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("100000000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("011111110")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("100000000")));
 
         // zero tail
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("111111000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("111111000")));
 
         // aggregation group & zero tail
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("110000111")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("110111000")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("111110111")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("111110001")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("111110100")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("110000100")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("110000111")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("110111000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("111110111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("111110001")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("111110100")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("110000100")));
     }
 
     @Test
@@ -127,25 +127,25 @@ public class CuboidTest extends LocalFileMetadataTestCase {
         CubeDesc cube = getTestKylinCubeWithoutSeller();
         CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("111111111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("111111111")));
 
         // base
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, 0));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("11111111")));
+        assertEquals(false, cuboidScheduler.isValid(0));
+        assertEquals(true, cuboidScheduler.isValid(toLong("11111111")));
 
         // aggregation group & zero tail
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000111")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10001111")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("11001111")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000001")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000101")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10001111")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("11001111")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000001")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000101")));
 
         // hierarchy
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10100000")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10110000")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10111000")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10001000")));
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10011000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10100000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10110000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10111000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10001000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10011000")));
     }
 
     @Test
@@ -153,11 +153,11 @@ public class CuboidTest extends LocalFileMetadataTestCase {
         CubeDesc cube = getSSBCubeDesc();
         CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
 
-        assertEquals(false, Cuboid.isValid(cuboidScheduler, toLong("10000000000")));
+        assertEquals(false, cuboidScheduler.isValid(toLong("10000000000")));
 
         // the 4th is mandatory and isMandatoryOnlyValid is true
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("10000001000")));
-        assertEquals(true, Cuboid.isValid(cuboidScheduler, toLong("00000001000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("10000001000")));
+        assertEquals(true, cuboidScheduler.isValid(toLong("00000001000")));
     }
 
     @Test


[12/22] kylin git commit: APACHE-KYLIN-2732: Introduce base cuboid as a new input for cubing job

Posted by nj...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 1e10d79..0338da8 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -26,7 +26,6 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
@@ -83,7 +82,7 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
     @Test
     public void test() throws Exception {
 
-        ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000);
+        ArrayBlockingQueue<String[]> queue = new ArrayBlockingQueue<String[]>(1000);
         ExecutorService executorService = Executors.newSingleThreadExecutor();
         long randSeed = System.currentTimeMillis();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/386a2ff7/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index 49d267a..ad754cd 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -113,7 +113,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
         //DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
         cubeBuilder.setConcurrentThreads(nThreads);
 
-        ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000);
+        ArrayBlockingQueue<String[]> queue = new ArrayBlockingQueue<String[]>(1000);
         ExecutorService executorService = Executors.newSingleThreadExecutor();
 
         try {
@@ -144,11 +144,13 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
         }
     }
 
-    static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<List<String>> queue, int count) throws IOException, InterruptedException {
+    static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<String[]> queue, int count)
+            throws IOException, InterruptedException {
         feedData(cube, flatTable, queue, count, 0);
     }
 
-    static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<List<String>> queue, int count, long randSeed) throws IOException, InterruptedException {
+    static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<String[]> queue, int count,
+            long randSeed) throws IOException, InterruptedException {
         IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
         int nColumns = flatDesc.getAllColumns().size();
 
@@ -177,14 +179,14 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
 
         // output with random data
         for (; count > 0; count--) {
-            ArrayList<String> row = new ArrayList<String>(nColumns);
+            String[] row = new String[nColumns];
             for (int i = 0; i < nColumns; i++) {
                 String[] candidates = distincts.get(i);
-                row.add(candidates[rand.nextInt(candidates.length)]);
+                row[i] = candidates[rand.nextInt(candidates.length)];
             }
             queue.put(row);
         }
-        queue.put(new ArrayList<String>(0));
+        queue.put(InputConverterUnitForRawData.EMPTY_ROW);
     }
 
     static Map<TblColRef, Dictionary<String>> getDictionaryMap(CubeInstance cube, String flatTable) throws IOException {