You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/08/11 05:24:40 UTC
[7/8] kylin git commit: KYLIN-2783 fix serialization
KYLIN-2783 fix serialization
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1cae1f5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1cae1f5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1cae1f5b
Branch: refs/heads/2.1.x
Commit: 1cae1f5b7ab879f3ec0735e9d176a142556e4d23
Parents: cfda465
Author: Li Yang <li...@apache.org>
Authored: Fri Aug 11 10:47:04 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Aug 11 12:17:54 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/cuboid/DefaultCuboidScheduler.java | 5 +----
.../org/apache/kylin/engine/spark/SparkCubingByLayer.java | 10 +++-------
2 files changed, 4 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1cae1f5b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index 859ad20..b75acd5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube.cuboid;
-import java.io.Serializable;
-
/**
*/
@@ -45,8 +43,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-@SuppressWarnings("serial")
-public class DefaultCuboidScheduler extends CuboidScheduler implements Serializable {
+public class DefaultCuboidScheduler extends CuboidScheduler {
private final long max;
private final Set<Long> allCuboidIds;
private final Map<Long, List<Long>> parent2child;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1cae1f5b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 2f54e25..544e072 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -44,7 +44,6 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.common.RowKeySplitter;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
import org.apache.kylin.cube.model.CubeDesc;
@@ -169,7 +168,6 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
final Broadcast<CubeSegment> vCubeSegment = sc.broadcast(cubeSegment);
final NDCuboidBuilder ndCuboidBuilder = new NDCuboidBuilder(vCubeSegment.getValue(), new RowKeyEncoderProvider(vCubeSegment.getValue()));
- final Broadcast<CuboidScheduler> vCuboidScheduler = sc.broadcast(vCubeDesc.getValue().getCuboidScheduler());
final int measureNum = cubeDesc.getMeasures().size();
int countMeasureIndex = 0;
@@ -259,7 +257,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
saveToHDFS(allRDDs[0], vCubeDesc.getValue(), outputPath, 0, confOverwrite);
// aggregate to ND cuboids
- PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), vCuboidScheduler.getValue(), ndCuboidBuilder);
+ PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), ndCuboidBuilder);
for (level = 1; level <= totalLevels; level++) {
partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
@@ -342,15 +340,13 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
CubeSegment cubeSegment;
CubeDesc cubeDesc;
- CuboidScheduler cuboidScheduler;
NDCuboidBuilder ndCuboidBuilder;
RowKeySplitter rowKeySplitter;
transient boolean initialized = false;
- CuboidFlatMap(CubeSegment cubeSegment, CubeDesc cubeDesc, CuboidScheduler cuboidScheduler, NDCuboidBuilder ndCuboidBuilder) {
+ CuboidFlatMap(CubeSegment cubeSegment, CubeDesc cubeDesc, NDCuboidBuilder ndCuboidBuilder) {
this.cubeSegment = cubeSegment;
this.cubeDesc = cubeDesc;
- this.cuboidScheduler = cuboidScheduler;
this.ndCuboidBuilder = ndCuboidBuilder;
this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
}
@@ -366,7 +362,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
long cuboidId = rowKeySplitter.split(key);
Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
- Collection<Long> myChildren = cuboidScheduler.getSpanningCuboid(cuboidId);
+ Collection<Long> myChildren = cubeDesc.getCuboidScheduler().getSpanningCuboid(cuboidId);
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {