You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/01/20 08:04:07 UTC
kylin git commit: Make integration test to building cube with spark
Repository: kylin
Updated Branches:
refs/heads/spark-it 36980c38b -> 387fa1458
Make integration test to building cube with spark
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/387fa145
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/387fa145
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/387fa145
Branch: refs/heads/spark-it
Commit: 387fa1458e950fb588aa38c006ba7fa514dd7f23
Parents: 36980c3
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jan 20 16:03:55 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jan 20 16:03:55 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 ++++++
.../measure/bitmap/RoaringBitmapCounter.java | 3 ++-
.../bitmap/RoaringBitmapCounterFactory.java | 3 ++-
.../measure/percentile/PercentileCounter.java | 22 +++++++++++++++++++-
.../apache/kylin/engine/spark/SparkCubing.java | 5 ++++-
kylin-it/pom.xml | 19 +++++++++++++++++
6 files changed, 54 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 35f5f7a..1734bee 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -68,6 +68,12 @@ abstract public class KylinConfigBase implements Serializable {
return sparkHome;
}
+ sparkHome = System.getProperty("SPARK_HOME");
+ if (StringUtils.isNotEmpty(sparkHome)) {
+ logger.info("SPARK_HOME was set to " + sparkHome);
+ return sparkHome;
+ }
+
return getKylinHome() + File.separator + "spark";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..3d6d3e4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -24,6 +24,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Iterator;
@@ -31,7 +32,7 @@ import java.util.Iterator;
/**
* A {@link BitmapCounter} based on roaring bitmap.
*/
-public class RoaringBitmapCounter implements BitmapCounter {
+public class RoaringBitmapCounter implements BitmapCounter, Serializable {
private ImmutableRoaringBitmap bitmap;
http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
index a71df95..822afa2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
@@ -21,9 +21,10 @@ package org.apache.kylin.measure.bitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
-public class RoaringBitmapCounterFactory implements BitmapCounterFactory {
+public class RoaringBitmapCounterFactory implements BitmapCounterFactory, Serializable {
public static final BitmapCounterFactory INSTANCE = new RoaringBitmapCounterFactory();
private RoaringBitmapCounterFactory() {}
http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
index bf505cf..18d3bba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
@@ -18,6 +18,9 @@
package org.apache.kylin.measure.percentile;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
@@ -30,7 +33,7 @@ public class PercentileCounter implements Serializable {
double compression;
double quantileRatio;
- TDigest registers;
+ transient TDigest registers;
public PercentileCounter(double compression) {
this(compression, INVALID_QUANTILE_RATIO);
@@ -94,4 +97,21 @@ public class PercentileCounter implements Serializable {
public void clear() {
reInitRegisters();
}
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.defaultWriteObject();
+ registers.compress();
+ int bound = registers.byteSize();
+ ByteBuffer buf = ByteBuffer.allocate(bound);
+ registers.asSmallBytes(buf);
+ out.writeInt(bound);
+ out.write(buf.array());
+ }
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ int bound = in.readInt();
+ ByteBuffer buf = ByteBuffer.allocate(bound);
+ in.read(buf.array());
+ registers = AVLTreeDigest.fromBytes(buf);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 0437a80..1a6ec70 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -630,7 +630,10 @@ public class SparkCubing extends AbstractApplication {
kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
- kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
try {
kyroClasses.add(Class.forName("com.google.common.collect.EmptyImmutableList"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 69e7909..5ea11a9 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -241,6 +241,25 @@
<artifactId>kafka_2.10</artifactId>
<scope>provided</scope>
</dependency>
+
+ <!-- Spark dependency -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.10</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive_2.10</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>