You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/13 23:33:27 UTC
kylin git commit: KYLIN-2428 Generate kyro mapping by Reflection
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2428 900436e36 -> 62463b741
KYLIN-2428 Generate kyro mapping by Reflection
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62463b74
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62463b74
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62463b74
Branch: refs/heads/KYLIN-2428
Commit: 62463b7413f34718ee29005c414b0b86bb751d8e
Parents: 900436e
Author: Billy Liu <bi...@apache.org>
Authored: Tue Feb 14 07:33:13 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Feb 14 07:33:13 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/JoinsTree.java | 2 +-
engine-spark/pom.xml | 7 +
.../engine/spark/KylinKryoRegistrator.java | 187 ++++++++++++++++++-
.../engine/spark/util/KyroMappingGenerator.java | 51 +++++
4 files changed, 245 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 3ab9a46..c7666cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -113,7 +113,7 @@ public class JoinsTree implements Serializable {
return chain.join;
}
- static class Chain implements Serializable {
+ public static class Chain implements Serializable {
private static final long serialVersionUID = 1L;
TableRef table; // pk side
http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 258232d..7a478f3 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -111,6 +111,13 @@
<artifactId>maven-model</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.10</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index c99ee6e..d9bfe99 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -61,7 +61,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
-// kyroClasses.add(Hashing.murmur3_128().getClass());
+ // kyroClasses.add(Hashing.murmur3_128().getClass());
kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
kyroClasses.add(scala.math.BigDecimal.class);
@@ -146,6 +146,9 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
+ kylinClassByReflection1(kyroClasses);
+ kylinClassByReflection2(kyroClasses);
+
kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class);
kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class);
kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class);
@@ -168,6 +171,188 @@ public class KylinKryoRegistrator implements KryoRegistrator {
// kryo.register(PercentileCounter.class, new JavaSerializer());
}
+ /**
+ * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods
+ * @param kyroClasses
+ */
+ private void kylinClassByReflection1(Set<Class> kyroClasses) {
+ kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
+ kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class);
+ kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class);
+ kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class);
+ kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
+ kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class);
+ kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureType.class);
+ kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class);
+ kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
+ kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class);
+ kyroClasses.add(org.apache.kylin.common.util.Array.class);
+ kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class);
+ kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class);
+ kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class);
+ kyroClasses.add(org.apache.kylin.common.util.ByteArray.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class);
+ kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class);
+ kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class);
+ kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class);
+ kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
+ kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class);
+ kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class);
+ kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class);
+ kyroClasses.add(org.apache.kylin.common.KylinConfig.class);
+ kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
+ kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.CubeInstance.class);
+ }
+
+ /**
+ * Generated by KyroMappingGenerator
+ * @param kyroClasses
+ */
+ private void kylinClassByReflection2(Set<Class> kyroClasses) {
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class);
+ kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class);
+ kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.CubeSegment.class);
+ kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class);
+ kyroClasses.add(org.apache.kylin.common.util.Pair.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class);
+ kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class);
+ kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class);
+ kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class);
+ kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class);
+ kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+ kyroClasses.add(org.apache.kylin.common.util.Dictionary.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class);
+ kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class);
+ kyroClasses.add(org.apache.kylin.job.JobInstance.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class);
+ kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class);
+ kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class);
+ kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class);
+ kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class);
+ kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class);
+ kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class);
+ kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class);
+ kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class);
+ kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class);
+ kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class);
+ kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class);
+ }
+
private static void addClassQuitely(Set<Class> kyroClasses, String className) {
try {
kyroClasses.add(Class.forName(className));
http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
new file mode 100644
index 0000000..b4960e9
--- /dev/null
+++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark.util;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.measure.MeasureIngester;
+import org.reflections.Reflections;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Generate Kyro Registrator class
+ */
+public class KyroMappingGenerator {
+ public static void main(String[] args) {
+ Set<Class<? extends Serializable>> subTypesOfSerializable = new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class);
+ String begin = "kyroClasses.add(";
+ String end = ".class);";
+ for (Class clazz : subTypesOfSerializable) {
+ if (clazz.getCanonicalName() != null)
+ System.out.println(begin + clazz.getCanonicalName() + end);
+ }
+ Set<Class<? extends BytesSerializer>> subTypesOfBytes = new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(BytesSerializer.class);
+ for (Class clazz : subTypesOfBytes) {
+ if (clazz.getCanonicalName() != null)
+ System.out.println(begin + clazz.getCanonicalName() + end);
+ }
+ Set<Class<? extends MeasureIngester>> subTypesOfMeasure = new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class);
+ for (Class clazz : subTypesOfMeasure) {
+ if (clazz.getCanonicalName() != null)
+ System.out.println(begin + clazz.getCanonicalName() + end);
+ }
+ }
+}