You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/18 16:52:45 UTC
[1/2] kylin git commit: KYLIN-2428 Cleanup unnecessary shaded
libraries for job/coprocessor/jdbc/server
Repository: kylin
Updated Branches:
refs/heads/master a7126f00c -> 4f9400108
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 3d33aa8..20b39ee 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -19,15 +19,11 @@
package org.apache.kylin.engine.spark;
import com.esotericsoftware.kryo.Kryo;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-import org.apache.kylin.measure.MeasureIngester;
import org.apache.spark.serializer.KryoRegistrator;
-import org.reflections.Reflections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Serializable;
+import java.util.LinkedHashSet;
import java.util.Set;
/**
@@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
@Override
public void registerClasses(Kryo kryo) {
- Set<Class> kyroClasses = Sets.newLinkedHashSet();
+ Set<Class> kyroClasses = new LinkedHashSet();
kyroClasses.add(byte[].class);
kyroClasses.add(int[].class);
kyroClasses.add(byte[][].class);
@@ -55,15 +51,11 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(java.util.HashMap.class);
kyroClasses.add(java.util.TreeMap.class);
kyroClasses.add(java.util.Properties.class);
- kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class));
- kyroClasses.addAll(new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class));
+ kyroClasses.add(java.math.MathContext.class);
+ kyroClasses.add(java.math.RoundingMode.class);
+ kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
+ kyroClasses.add(java.util.Random.class);
+ kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class);
kyroClasses.add(org.apache.spark.sql.Row[].class);
kyroClasses.add(org.apache.spark.sql.Row.class);
@@ -74,65 +66,29 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
- kyroClasses.add(Hashing.murmur3_128().getClass());
kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
kyroClasses.add(scala.math.BigDecimal.class);
- kyroClasses.add(java.math.MathContext.class);
- kyroClasses.add(java.math.RoundingMode.class);
- kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
- kyroClasses.add(java.util.Random.class);
- kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class);
kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class);
- kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
- kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
- kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
- kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
- kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
- kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class);
kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class);
kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class);
- kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class);
kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class);
- kyroClasses.add(org.apache.kylin.common.util.Array.class);
- kyroClasses.add(org.apache.kylin.metadata.model.Segments.class);
- kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class);
- kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class);
- kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
- kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
- kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
- kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class);
- kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class);
- kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
- kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
- kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
- kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
- kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
- kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
- kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
- kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
- kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
- kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
- kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
- kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+
+ kylinClassByReflection1(kyroClasses);
+ kylinClassByReflection2(kyroClasses);
+
+ kyroClasses.add(com.google.common.hash.Hashing.murmur3_128().getClass());
kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class);
kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class);
kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class);
kyroClasses.add(org.roaringbitmap.buffer.MappeableArrayContainer.class);
- kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
- kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
- kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class);
- kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
kyroClasses.add(com.tdunning.math.stats.AVLTreeDigest.class);
kyroClasses.add(com.tdunning.math.stats.Centroid.class);
@@ -151,6 +107,195 @@ public class KylinKryoRegistrator implements KryoRegistrator {
// kryo.register(PercentileCounter.class, new JavaSerializer());
}
+ /**
+ * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods
+ * @param kyroClasses
+ */
+ private void kylinClassByReflection1(Set<Class> kyroClasses) {
+ kyroClasses.add(org.apache.kylin.common.KylinConfig.class);
+ kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class);
+ kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class);
+ kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class);
+ kyroClasses.add(org.apache.kylin.common.util.Array.class);
+ kyroClasses.add(org.apache.kylin.common.util.ByteArray.class);
+ kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class);
+ kyroClasses.add(org.apache.kylin.common.util.Dictionary.class);
+ kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class);
+ kyroClasses.add(org.apache.kylin.common.util.Pair.class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
+ kyroClasses.add(org.apache.kylin.cube.CubeInstance.class);
+ kyroClasses.add(org.apache.kylin.cube.CubeSegment.class);
+ kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class);
+ kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class);
+ kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class);
+ kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
+ kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class);
+ kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class);
+ kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class);
+ kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class);
+ kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class);
+ kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class);
+ kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class);
+ kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class);
+ kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class);
+ kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class);
+ kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class);
+ kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class);
+ kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class);
+ kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class);
+ kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class);
+ kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class);
+ kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.BooleanSerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class);
+ kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
+ kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class);
+ kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class);
+ kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class);
+ kyroClasses.add(org.apache.kylin.job.JobInstance.class);
+ kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class);
+ kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class);
+ }
+
+ /**
+ * Generated by KyroMappingGenerator
+ * @param kyroClasses
+ */
+ private void kylinClassByReflection2(Set<Class> kyroClasses) {
+ kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureTypeFactory.NeedRewriteOnlyMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
+ kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HLLDistinctCountAggFunc.FixedValueHLLCMockup.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class);
+ kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class);
+ kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class);
+ kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class);
+ kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.Segments.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class);
+ kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class);
+ kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class);
+ kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class);
+ kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class);
+ kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class);
+ kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class);
+ }
+
private static void addClassQuitely(Set<Class> kyroClasses, String className) {
try {
kyroClasses.add(Class.forName(className));
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 327d215..f68194e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -18,8 +18,8 @@
package org.apache.kylin.engine.spark;
-import org.apache.hadoop.util.ClassUtil;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.EngineFactory;
@@ -53,13 +53,14 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
StringBuilder jars = new StringBuilder();
- StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
- StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration", null)); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace", null)); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration", null)); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection", null)); // hbase-client.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration", null)); // hbase-common.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer", null)); // hbase-protocol.jar
+ StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge", null)); // metrics-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps", "guava")); //guava.jar
StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
sparkExecutable.setJars(jars.toString());
@@ -73,9 +74,9 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
}
- private String findJar(String className) {
+ private String findJar(String className, String perferLibraryName) {
try {
- return ClassUtil.findContainingJar(Class.forName(className));
+ return ClassUtil.findContainingJar(Class.forName(className), perferLibraryName);
} catch (ClassNotFoundException e) {
logger.warn("failed to locate jar for class " + className + ", ignore it", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index c7422ed..f7ed2d0 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -17,7 +17,6 @@
*/
package org.apache.kylin.engine.spark;
-import com.google.common.collect.Lists;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -72,6 +71,7 @@ import java.io.File;
import java.io.FileFilter;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -333,7 +333,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
}
- private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = Lists.newArrayListWithCapacity(0);
+ private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = new ArrayList(0);
class CuboidFlatMap implements PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> {
@@ -370,7 +370,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
return EMTPY_ITERATOR;
}
- List<Tuple2<ByteArray, Object[]>> tuples = Lists.newArrayListWithCapacity(myChildren.size());
+ List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size());
for (Long child : myChildren) {
Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 733a472..015b480 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -20,8 +20,7 @@ package org.apache.kylin.engine.spark;
import java.io.File;
import java.util.Map;
-import jodd.util.StringUtil;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Logger;
@@ -31,8 +30,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-
/**
*/
public class SparkExecutable extends AbstractExecutable {
@@ -74,12 +71,16 @@ public class SparkExecutable extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final KylinConfig config = context.getConfig();
- Preconditions.checkNotNull(config.getSparkHome());
- Preconditions.checkNotNull(config.getKylinJobJarPath());
+ if (config.getSparkHome() == null) {
+ throw new NullPointerException();
+ }
+ if (config.getKylinJobJarPath() == null) {
+ throw new NullPointerException();
+ }
String jars = this.getParam(JARS);
String hadoopConf = "/etc/hadoop/conf";
- if (StringUtil.isNotEmpty(config.getHadoopConfDir())) {
+ if (StringUtils.isNotEmpty(config.getHadoopConfDir())) {
hadoopConf = config.getHadoopConfDir();
} else {
String hiveConf = ClassLoader.getSystemClassLoader().getResource("hive-site.xml").getFile().toString();
@@ -104,7 +105,7 @@ public class SparkExecutable extends AbstractExecutable {
}
StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.common.util.SparkEntry ");
+ stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.engine.spark.util.SparkEntry ");
Map<String, String> sparkConfs = config.getSparkConfigOverride();
for (Map.Entry<String, String> entry : sparkConfs.entrySet()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
index 531670f..a2740bf 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
@@ -23,8 +23,6 @@ import java.util.ArrayList;
import org.apache.kylin.engine.spark.SparkCuboidWriter;
import org.apache.kylin.gridtable.GTRecord;
-import com.google.common.collect.Lists;
-
import scala.Tuple2;
/**
@@ -35,7 +33,7 @@ public class ListBackedCuboidWriter implements SparkCuboidWriter {
private final TupleConverter tupleConverter;
public ListBackedCuboidWriter(TupleConverter tupleConverter) {
- this.result = Lists.newArrayList();
+ this.result = new ArrayList();
this.tupleConverter = tupleConverter;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
index 13464a0..a8a4d28 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
@@ -24,8 +24,6 @@ import java.util.NoSuchElementException;
import org.apache.spark.api.java.function.Function;
-import com.google.common.collect.Lists;
-
import scala.Tuple2;
/**
@@ -47,7 +45,7 @@ public class IteratorUtils {
if (!hasNext()) {
throw new NoSuchElementException();
}
- final LinkedList<V> values = Lists.newLinkedList();
+ final LinkedList<V> values = new LinkedList();
K currentKey = current._1();
values.add(current._2());
while (input.hasNext()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
new file mode 100644
index 0000000..14788fc
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.engine.spark.util;
+
+import org.apache.commons.lang.StringUtils;
+
+import org.apache.kylin.common.util.AbstractApplication;
+
+/**
+ */
+public final class SparkEntry {
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
+ if (!(args.length >= 2)) {
+ throw new IllegalArgumentException(String.valueOf("-className is required"));
+ }
+ if (!(args[0].equals("-className"))) {
+ throw new IllegalArgumentException(String.valueOf("-className is required"));
+ }
+ final String className = args[1];
+ final Object o = Class.<AbstractApplication> forName(className).newInstance();
+ if (!(o instanceof AbstractApplication)) {
+ throw new IllegalArgumentException(String.valueOf(className + " is not a subClass of AbstractSparkApplication"));
+ }
+ String[] appArgs = new String[args.length - 2];
+ for (int i = 2; i < args.length; i++) {
+ appArgs[i - 2] = args[i];
+ }
+ AbstractApplication abstractApplication = (AbstractApplication) o;
+ abstractApplication.execute(appArgs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
new file mode 100644
index 0000000..b181d33
--- /dev/null
+++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark.util;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.measure.MeasureIngester;
+import org.reflections.Reflections;
+
+import java.io.Serializable;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Generate Kyro Registrator class, the output will be added into KylinKyroRegistrator manually. No runtime dependency with Reflections.
+ */
+public class KyroMappingGenerator {
+ public static void main(String[] args) {
+ Set<Class<? extends Serializable>> subTypesOfSerializable = new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class);
+ String begin = "kyroClasses.add(";
+ String end = ".class);";
+ TreeSet<String> sortedSet = new TreeSet();
+ for (Class clazz : subTypesOfSerializable) {
+ if (clazz.getCanonicalName() != null)
+ sortedSet.add(clazz.getCanonicalName());
+ }
+ Set<Class<? extends BytesSerializer>> subTypesOfBytes = new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(BytesSerializer.class);
+ for (Class clazz : subTypesOfBytes) {
+ if (clazz.getCanonicalName() != null)
+ sortedSet.add(clazz.getCanonicalName());
+ }
+ Set<Class<? extends MeasureIngester>> subTypesOfMeasure = new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class);
+ for (Class clazz : subTypesOfMeasure) {
+ if (clazz.getCanonicalName() != null)
+ sortedSet.add(clazz.getCanonicalName());
+ }
+ for (String className : sortedSet) {
+ System.out.println(begin + className + end);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 874ead6..02890a6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -43,7 +43,7 @@
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica</artifactId>
+ <artifactId>avatica-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -69,9 +69,7 @@
<artifactSet>
<excludes>
- <exclude>com.google.protobuf:*</exclude>
- <exclude>commons-logging:*</exclude>
- <exclude>commons-codec:*</exclude>
+ <exclude>org.slf4j:jcl-over-slf4j:*</exclude>
</excludes>
</artifactSet>
@@ -85,9 +83,17 @@
<shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>${shadeBase}.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons</shadedPattern>
+ </relocation>
</relocations>
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b82eee2..a103ae0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,15 +55,34 @@
<!-- HBase versions -->
<hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+
+ <!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
- <!-- Hadoop deps, keep compatible with hadoop2.version -->
+ <!-- Spark versions -->
+ <spark.version>1.6.3</spark.version>
+ <kryo.version>4.0.0</kryo.version>
+ <!--
+ <reflections.version>0.9.10</reflections.version>
+ -->
+
+ <!-- Calcite Version -->
+ <calcite.version>1.11.0</calcite.version>
+ <avatica.version>1.9.0</avatica.version>
+
+ <!-- Hadoop Common deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
<curator.version>2.7.1</curator.version>
- <jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
+ <jsch.version>0.1.53</jsch.version>
<commons-cli.version>1.2</commons-cli.version>
+ <commons-lang.version>2.6</commons-lang.version>
+ <commons-io.version>2.5</commons-io.version>
+ <commons-math3.version>3.6.1</commons-math3.version>
+
+ <!-- Calcite deps, keep compatible with calcite.version -->
+ <jackson.version>2.6.3</jackson.version>
<!-- Test Dependency versions -->
<antlr.version>3.4</antlr.version>
@@ -75,26 +94,16 @@
<jamm.version>0.3.1</jamm.version>
<!-- Commons -->
- <commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-collections.version>3.2.2</commons-collections.version>
- <commons-io.version>2.5</commons-io.version>
<commons-daemon.version>1.0.15</commons-daemon.version>
<commons-email.version>1.4</commons-email.version>
- <commons-math3.version>3.6.1</commons-math3.version>
-
- <!-- Spark -->
- <spark.version>1.6.3</spark.version>
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.21</slf4j.version>
- <reflections.version>0.9.10</reflections.version>
- <jsch.version>0.1.53</jsch.version>
<xerces.version>2.11.0</xerces.version>
<xalan.version>2.7.2</xalan.version>
- <extendedset.version>1.3.4</extendedset.version>
- <kryo.version>4.0.0</kryo.version>
<ehcache.version>2.10.2.2.21</ehcache.version>
<apache-httpclient.version>4.2.5</apache-httpclient.version>
<roaring.version>0.6.18</roaring.version>
@@ -111,10 +120,6 @@
<opensaml.version>2.6.4</opensaml.version>
<aspectj.version>1.8.9</aspectj.version>
- <!-- Calcite Version -->
- <calcite.version>1.11.0</calcite.version>
- <avatica.version>1.9.0</avatica.version>
-
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
<sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
@@ -279,6 +284,11 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-tool-assembly</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-it</artifactId>
<version>${project.version}</version>
</dependency>
@@ -390,6 +400,66 @@
<version>${hadoop2.version}</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>${jsr305.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>${jsch.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>${commons-cli.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>${commons-math3.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- HBase2 dependencies -->
<dependency>
@@ -461,6 +531,11 @@
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ <version>${avatica.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
<version>${avatica.version}</version>
</dependency>
@@ -476,23 +551,39 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo-shaded</artifactId>
+ <version>${kryo.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Kafka dependency -->
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>${kafka.version}</version>
@@ -501,26 +592,6 @@
<!-- Other dependencies -->
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- <version>${commons-cli.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${commons-lang.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
@@ -531,21 +602,15 @@
<version>${commons-collections.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>${commons-math3.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons-io.version}</version>
- </dependency>
- <dependency>
<groupId>commons-daemon</groupId>
<artifactId>commons-daemon</artifactId>
<version>${commons-daemon.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-email</artifactId>
+ <version>${commons-email.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
@@ -571,40 +636,18 @@
<version>${slf4j.version}</version>
</dependency>
+ <!-- Test -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-email</artifactId>
- <version>${commons-email.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>${jsr305.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>${reflections.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- <version>${jsch.version}</version>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.dbunit</groupId>
<artifactId>dbunit</artifactId>
<version>${dbunit.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
@@ -615,6 +658,7 @@
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>xerces</groupId>
@@ -627,31 +671,10 @@
<version>${xalan.version}</version>
</dependency>
<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
- <version>${extendedset.version}</version>
- </dependency>
- <dependency>
- <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo-shaded</artifactId>
- <version>${kryo.version}</version>
- </dependency>
- <dependency>
<groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
<version>${jamm.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -1036,6 +1059,7 @@
<module>jdbc</module>
<module>assembly</module>
<module>tool</module>
+ <module>tool-assembly</module>
<module>kylin-it</module>
<module>tomcat-ext</module>
</modules>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 6ab74a7..d35c74c 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -47,10 +47,16 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
</dependency>
+
+ <!-- Provided -->
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 454ed6b..3c96e60 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -233,6 +233,10 @@
<artifactId>jetty-all</artifactId>
<groupId>org.eclipse.jetty.aggregate</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>calcite-core</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index e2fe448..da5e033 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -35,23 +35,19 @@
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-mr</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
+ <artifactId>kylin-engine-mr</artifactId>
</dependency>
+ <!-- Provided -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
</dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -76,9 +72,5 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
-
-
-
</dependencies>
-
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 3aea531..29ca7e5 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -32,8 +32,11 @@
<version>2.0.0-SNAPSHOT</version>
</parent>
- <dependencies>
+ <properties>
+ <shadeBase>org.apache.kylin.coprocessor.shaded</shadeBase>
+ </properties>
+ <dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-mr</artifactId>
@@ -119,14 +122,20 @@
<include>org.apache.kylin:kylin-core-metadata</include>
<include>org.apache.kylin:kylin-core-dictionary</include>
<include>org.apache.kylin:kylin-core-cube</include>
- <include>com.ning:compress-lzf</include>
<include>org.roaringbitmap:RoaringBitmap</include>
<include>com.tdunning:t-digest</include>
- <!-- below for inverted index only -->
- <include>com.n3twork.druid:extendedset</include>
- <include>org.apache.commons:commons-lang3</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.roaringbitmap</pattern>
+ <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
+ </relocation>
+ </relocations>
<filters>
<filter>
<artifact>*:*</artifact>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
new file mode 100644
index 0000000..67d92d4
--- /dev/null
+++ b/tool-assembly/pom.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-tool-assembly</artifactId>
+ <name>Apache Kylin - Tool Assembly</name>
+ <packaging>jar</packaging>
+ <description>Apache Kylin - Tool Assembly</description>
+
+ <parent>
+ <artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <shadeBase>org.apache.kylin.tool.shaded</shadeBase>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-tool</artifactId>
+ </dependency>
+
+ <!-- As KylinConfigCLI will be called before Hbase/Hive/Hadoop dependency loaded, has to define the following as compile in tool -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>assembly</shadedClassifierName>
+ <artifactSet>
+ <includes>
+ <!-- shade the httpcore to avoid the lower version conflict with HBase one -->
+ <include>org.apache.httpcomponents:httpcore</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ <include>log4j:log4j</include>
+ <include>commons-io:commons-io</include>
+ <include>commons-lang:commons-lang</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.kylin:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>${shadeBase}.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.io</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.io</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.lang</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.lang</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+ <shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 91040d4..278c2b8 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -81,43 +81,4 @@
</dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>assembly</shadedClassifierName>
- <artifactSet>
- <includes>
- <!-- shade the httpcore to avoid the lower version conflict with HBase one -->
- <include>org.apache.httpcomponents:httpcore</include>
- <include>org.apache.kylin:kylin-tool</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
</project>
[2/2] kylin git commit: KYLIN-2428 Cleanup unnecessary shaded
libraries for job/coprocessor/jdbc/server
Posted by bi...@apache.org.
KYLIN-2428 Cleanup unnecessary shaded libraries for job/coprocessor/jdbc/server
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f940010
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f940010
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f940010
Branch: refs/heads/master
Commit: 4f940010894a3ee7c284ded15e1d7ef6cc16e966
Parents: a7126f0
Author: Billy Liu <bi...@apache.org>
Authored: Sun Feb 19 00:52:30 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Sun Feb 19 00:52:30 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 49 +++-
atopcalcite/pom.xml | 1 +
build/bin/get-properties.sh | 3 +-
build/bin/kylin.sh | 8 +-
build/conf/kylin-tools-log4j.properties | 9 +-
build/script/prepare_libs.sh | 2 +-
core-common/pom.xml | 22 +-
.../org/apache/kylin/common/util/ClassUtil.java | 50 ++++
.../apache/kylin/common/util/SparkEntry.java | 42 ---
.../apache/kylin/common/util/ClassUtilTest.java | 32 +++
core-cube/pom.xml | 18 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../apache/kylin/gridtable/GTInvertedIndex.java | 223 ----------------
.../gridtable/GTInvertedIndexOfColumn.java | 133 ----------
.../gridtable/SimpleInvertedIndexTest.java | 196 --------------
core-job/pom.xml | 9 +
core-metadata/pom.xml | 23 ++
.../apache/kylin/dimension/BooleanDimEnc.java | 5 +-
.../org/apache/kylin/dimension/DateDimEnc.java | 4 +-
.../kylin/dimension/DictionaryDimEnc.java | 3 +-
.../apache/kylin/dimension/FixedLenDimEnc.java | 3 +-
.../kylin/dimension/FixedLenHexDimEnc.java | 3 +-
.../org/apache/kylin/dimension/IntDimEnc.java | 3 +-
.../apache/kylin/dimension/IntegerDimEnc.java | 3 +-
.../kylin/dimension/OneMoreByteVLongDimEnc.java | 3 +-
.../org/apache/kylin/dimension/TimeDimEnc.java | 4 +-
.../kylin/measure/MeasureTypeFactory.java | 2 +-
.../measure/hllc/HLLDistinctCountAggFunc.java | 2 +-
.../org/apache/kylin/measure/topn/Counter.java | 3 +-
.../kylin/metadata/model/DataModelDesc.java | 5 +-
.../apache/kylin/metadata/model/JoinsTree.java | 2 +-
.../kylin/metadata/model/PartitionDesc.java | 2 +-
.../kylin/metadata/model/SegmentStatusEnum.java | 4 +-
.../apache/kylin/metadata/model/Segments.java | 3 +-
.../realization/RealizationStatusEnum.java | 4 +-
core-storage/pom.xml | 20 ++
engine-spark/pom.xml | 47 +---
.../engine/spark/KylinKryoRegistrator.java | 265 ++++++++++++++-----
.../spark/SparkBatchCubingJobBuilder2.java | 21 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 6 +-
.../kylin/engine/spark/SparkExecutable.java | 17 +-
.../spark/cube/ListBackedCuboidWriter.java | 4 +-
.../kylin/engine/spark/util/IteratorUtils.java | 4 +-
.../kylin/engine/spark/util/SparkEntry.java | 48 ++++
.../engine/spark/util/KyroMappingGenerator.java | 56 ++++
jdbc/pom.xml | 14 +-
pom.xml | 216 ++++++++-------
query/pom.xml | 6 +
server/pom.xml | 4 +
source-kafka/pom.xml | 16 +-
storage-hbase/pom.xml | 19 +-
tool-assembly/pom.xml | 135 ++++++++++
tool/pom.xml | 39 ---
53 files changed, 883 insertions(+), 934 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 873abff..ba69583 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,16 +19,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-assembly</artifactId>
+ <name>Apache Kylin - Assembly</name>
+
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-assembly</artifactId>
- <name>Apache Kylin - Assembly</name>
+ <properties>
+ <shadeBase>org.apache.kylin.job.shaded</shadeBase>
+ </properties>
<dependencies>
<dependency>
@@ -180,8 +184,45 @@
<exclude>org.apache.zookeeper:*</exclude>
<exclude>net.sf.ehcache:*</exclude>
<exclude>org.apache.httpcomponents:*</exclude>
+ <exclude>org.apache.commons:commons-email:*</exclude>
+ <exclude>com.sun.mail:javax.mail:*</exclude>
+ <exclude>javax.activation:activation:*</exclude>
</excludes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.codec</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.codec</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.collections</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.collections</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.lang3</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.lang3</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.logging</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.logging</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.roaringbitmap</pattern>
+ <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
+ </relocation>
+ </relocations>
<filters>
<filter>
<artifact>*:*</artifact>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop -->
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..57721d8 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -25,7 +25,6 @@ then
exit -1
fi
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
+result=`java -cp $tool_jar -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
echo "$result"
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index cc571e5..15e1aed 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,7 +38,7 @@ function retrieveDependency() {
source ${dir}/setenv.sh
fi
- export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+ export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
if [ -n "$KAFKA_HOME" ]
then
@@ -103,7 +103,7 @@ then
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
- -Dlog4j.configuration=kylin-server-log4j.properties \
+ -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
-Djava.endorsed.dirs=${tomcat_root}/endorsed \
@@ -165,8 +165,10 @@ then
then source ${dir}/setenv-tool.sh
fi
+ hbase_original=${HBASE_CLASSPATH}
+ export HBASE_CLASSPATH=${hbase_original}:${KYLIN_HOME}/tool/*
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties "$@"
-
+ export HBASE_CLASSPATH=${hbase_original}
else
quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/build/conf/kylin-tools-log4j.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index 0ba7d2e..e975d18 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -25,11 +25,12 @@
# It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties
# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=INFO,stderr
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
#log4j.logger.org.apache.hadoop=ERROR
log4j.logger.org.apache.kylin=DEBUG
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/build/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh
index aec9923..3dbdf21 100755
--- a/build/script/prepare_libs.sh
+++ b/build/script/prepare_libs.sh
@@ -33,7 +33,7 @@ mkdir build/lib build/tool
cp assembly/target/kylin-assembly-${version}-job.jar build/lib/kylin-job-${version}.jar
cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar build/lib/kylin-coprocessor-${version}.jar
cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar
-cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
+cp tool-assembly/target/kylin-tool-assembly-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
# Copied file becomes 000 for some env (e.g. my Cygwin)
chmod 644 build/lib/kylin-job-${version}.jar
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
<dependencies>
<!-- Basic Utilities -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
</dependency>
@@ -63,9 +47,11 @@
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
</dependency>
+
+ <!-- Provided -->
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 3c63c41..9af34c3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -19,9 +19,12 @@
package org.apache.kylin.common.util;
import java.io.File;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
+import java.net.URLDecoder;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.WeakHashMap;
@@ -89,4 +92,51 @@ public class ClassUtil {
}
}
+ public static String findContainingJar(Class<?> clazz) {
+ return findContainingJar(clazz, null);
+ }
+
+ /**
+ * Load the first jar library contains clazz with preferJarKeyword matched. If preferJarKeyword is null, just load the
+ * jar likes Hadoop Commons' ClassUtil
+ * @param clazz
+ * @param preferJarKeyWord
+ * @return
+ */
+ public static String findContainingJar(Class<?> clazz, String preferJarKeyWord) {
+ ClassLoader loader = clazz.getClassLoader();
+ String classFile = clazz.getName().replaceAll("\\.", "/") + ".class";
+
+ try {
+ Enumeration e = loader.getResources(classFile);
+
+ URL url = null;
+ do {
+ if (!e.hasMoreElements()) {
+ if (url == null)
+ return null;
+ else
+ break;
+ }
+
+ url = (URL) e.nextElement();
+ if (!"jar".equals(url.getProtocol()))
+ break;
+ if (preferJarKeyWord != null && url.getPath().indexOf(preferJarKeyWord) != -1)
+ break;
+ if (preferJarKeyWord == null)
+ break;
+ } while (true);
+
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ return toReturn.replaceAll("!.*$", "");
+ } catch (IOException var6) {
+ throw new RuntimeException(var6);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
deleted file mode 100644
index fd324a2..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.util;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.google.common.base.Preconditions;
-
-/**
- */
-public final class SparkEntry {
-
- public static void main(String[] args) throws Exception {
- System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
- Preconditions.checkArgument(args.length >= 2, "-className is required");
- Preconditions.checkArgument(args[0].equals("-className"), "-className is required");
- final String className = args[1];
- final Object o = Class.<AbstractApplication> forName(className).newInstance();
- Preconditions.checkArgument(o instanceof AbstractApplication, className + " is not a subClass of AbstractSparkApplication");
- String[] appArgs = new String[args.length - 2];
- for (int i = 2; i < args.length; i++) {
- appArgs[i - 2] = args[i];
- }
- AbstractApplication abstractApplication = (AbstractApplication) o;
- abstractApplication.execute(appArgs);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
new file mode 100644
index 0000000..75fa574
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ClassUtilTest {
+
+ @Test
+ public void testFindContainingJar() throws ClassNotFoundException {
+ Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.beanutils.BeanUtils")).contains("commons-beanutils"));
+ Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.beanutils.BeanUtils"), "core").contains("commons-beanutils-core"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
+
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 9acb90c..0c1419d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -97,7 +97,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
}
- public enum DeriveType {
+ public enum DeriveType implements java.io.Serializable{
LOOKUP, PK_FK, EXTENDED_COLUMN
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
deleted file mode 100644
index a2c713a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * A thread-safe inverted index of row blocks in memory.
- *
- * Note function not() must return all blocks, because index only know what block contains a value,
- * but not sure what block does not contain a value.
- *
- * @author yangli9
- */
-public class GTInvertedIndex {
-
- private final GTInfo info;
- private final ImmutableBitSet colPreferIndex;
- private final ImmutableBitSet colBlocks;
- private final GTInvertedIndexOfColumn[] index; // for each column
-
- private volatile int nIndexedBlocks;
-
- public GTInvertedIndex(GTInfo info) {
- this.info = info;
- this.colPreferIndex = info.colPreferIndex;
- this.colBlocks = info.selectColumnBlocks(colPreferIndex);
-
- index = new GTInvertedIndexOfColumn[info.getColumnCount()];
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator());
- }
- }
-
- public void add(GTRowBlock block) {
-
- @SuppressWarnings("unchecked")
- Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- distinctValues[c] = new HashSet<ByteArray>();
- }
-
- GTRowBlock.Reader reader = block.getReader(colBlocks);
- GTRecord record = new GTRecord(info);
- while (reader.hasNext()) {
- reader.fetchNext(record);
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- distinctValues[c].add(record.get(c));
- }
- }
-
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- index[c].add(distinctValues[c], block.getSequenceId());
- }
-
- nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
- }
-
- public ConciseSet filter(TupleFilter filter) {
- return filter(filter, nIndexedBlocks);
- }
-
- public ConciseSet filter(TupleFilter filter, int totalBlocks) {
- // number of indexed blocks may increase as we do evaluation
- int indexedBlocks = nIndexedBlocks;
-
- Evaluator evaluator = new Evaluator(indexedBlocks);
- ConciseSet r = evaluator.evaluate(filter);
-
- // add blocks that have not been indexed
- for (int i = indexedBlocks; i < totalBlocks; i++) {
- r.add(i);
- }
-
- return r;
- }
-
- private class Evaluator {
- private int indexedBlocks;
-
- Evaluator(int indexedBlocks) {
- this.indexedBlocks = indexedBlocks;
- }
-
- public ConciseSet evaluate(TupleFilter filter) {
- if (filter == null) {
- return all();
- }
-
- if (filter instanceof LogicalTupleFilter)
- return evalLogical((LogicalTupleFilter) filter);
-
- if (filter instanceof CompareTupleFilter)
- return evalCompare((CompareTupleFilter) filter);
-
- // unable to evaluate
- return all();
- }
-
- @SuppressWarnings("unchecked")
- private ConciseSet evalCompare(CompareTupleFilter filter) {
- int col = col(filter);
- if (index[col] == null)
- return all();
-
- switch (filter.getOperator()) {
- case ISNULL:
- return index[col].getNull();
- case ISNOTNULL:
- return all();
- case EQ:
- return index[col].getEquals((ByteArray) filter.getFirstValue());
- case NEQ:
- return all();
- case IN:
- return index[col].getIn((Iterable<ByteArray>) filter.getValues());
- case NOTIN:
- return all();
- case LT:
- return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false);
- case LTE:
- return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true);
- case GT:
- return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false);
- case GTE:
- return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false);
- default:
- throw new IllegalStateException("Unsupported operator " + filter.getOperator());
- }
- }
-
- private ConciseSet evalLogical(LogicalTupleFilter filter) {
- List<? extends TupleFilter> children = filter.getChildren();
-
- switch (filter.getOperator()) {
- case AND:
- return evalLogicalAnd(children);
- case OR:
- return evalLogicalOr(children);
- case NOT:
- return evalLogicalNot(children);
- default:
- throw new IllegalStateException("Unsupported operator " + filter.getOperator());
- }
- }
-
- private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) {
- ConciseSet set = all();
-
- for (TupleFilter c : children) {
- ConciseSet t = evaluate(c);
- if (t == null)
- continue; // because it's AND
-
- set.retainAll(t);
- }
- return set;
- }
-
- private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) {
- ConciseSet set = new ConciseSet();
-
- for (TupleFilter c : children) {
- ConciseSet t = evaluate(c);
- if (t == null)
- return null; // because it's OR
-
- set.addAll(t);
- }
- return set;
- }
-
- private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) {
- return all();
- }
-
- private ConciseSet all() {
- return not(new ConciseSet());
- }
-
- private ConciseSet not(ConciseSet set) {
- set.add(indexedBlocks);
- set.complement();
- return set;
- }
-
- private int col(CompareTupleFilter filter) {
- return filter.getColumn().getColumnDesc().getZeroBasedIndex();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
deleted file mode 100644
index bfacc0f..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.NavigableMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import com.google.common.collect.Maps;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class GTInvertedIndexOfColumn {
-
- final private IGTComparator comparator;
- final private ReentrantReadWriteLock rwLock;
-
- private int nBlocks;
- private NavigableMap<ByteArray, ConciseSet> rangeIndex;
- private ConciseSet nullIndex;
-
- public GTInvertedIndexOfColumn(IGTComparator comparator) {
- this.comparator = comparator;
- this.rwLock = new ReentrantReadWriteLock();
- this.rangeIndex = Maps.newTreeMap(comparator);
- this.nullIndex = new ConciseSet();
- }
-
- public void add(Iterable<ByteArray> codes, int blockId) {
- rwLock.writeLock().lock();
- try {
- for (ByteArray code : codes) {
- if (comparator.isNull(code)) {
- nullIndex.add(blockId);
- continue;
- }
- ConciseSet set = rangeIndex.get(code);
- if (set == null) {
- set = new ConciseSet();
- rangeIndex.put(code.copy(), set);
- }
- set.add(blockId);
- }
-
- if (blockId >= nBlocks) {
- nBlocks = blockId + 1;
- }
-
- } finally {
- rwLock.writeLock().unlock();
- }
- }
-
- public ConciseSet getNull() {
- rwLock.readLock().lock();
- try {
- return nullIndex.clone();
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getEquals(ByteArray code) {
- rwLock.readLock().lock();
- try {
- ConciseSet set = rangeIndex.get(code);
- if (set == null)
- return new ConciseSet();
- else
- return set.clone();
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getIn(Iterable<ByteArray> codes) {
- rwLock.readLock().lock();
- try {
- ConciseSet r = new ConciseSet();
- for (ByteArray code : codes) {
- ConciseSet set = rangeIndex.get(code);
- if (set != null)
- r.addAll(set);
- }
- return r;
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) {
- rwLock.readLock().lock();
- try {
- ConciseSet r = new ConciseSet();
- if (from == null && to == null) {
- r.add(nBlocks);
- r.complement();
- return r;
- }
- NavigableMap<ByteArray, ConciseSet> subMap;
- if (from == null) {
- subMap = rangeIndex.headMap(to, toInclusive);
- } else if (to == null) {
- subMap = rangeIndex.tailMap(from, fromInclusive);
- } else {
- subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive);
- }
- for (ConciseSet set : subMap.values()) {
- r.addAll(set);
- }
- return r;
- } finally {
- rwLock.readLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
deleted file mode 100644
index 5e3e771..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
-
- GTInfo info;
- GTInvertedIndex index;
- ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
- ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
- @BeforeClass
- public static void setUp() throws Exception {
- staticCreateTestMetadata();
- }
-
- @AfterClass
- public static void after() throws Exception {
- cleanAfterClass();
- }
-
- public SimpleInvertedIndexTest() {
-
- info = UnitTestSupport.advancedInfo();
- TblColRef colA = info.colRef(0);
-
- // block i contains value "i", the last is NULL
- index = new GTInvertedIndex(info);
- GTRowBlock mockBlock = GTRowBlock.allocate(info);
- GTRowBlock.Writer writer = mockBlock.getWriter();
- GTRecord record = new GTRecord(info);
- for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
- for (int j = 0; j < info.getRowBlockSize(); j++) {
- writer.append(record);
- }
- writer.readyForFlush();
- index.add(mockBlock);
-
- writer.clearForNext();
- }
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
- basicResults.add(set(9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
- basicResults.add(set(0));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
- basicResults.add(set(0, 5));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
- basicResults.add(set(0, 1, 2));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
- basicResults.add(set(0, 1, 2, 3));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
- basicResults.add(set(4, 5, 6, 7, 8));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
- basicResults.add(set(3, 4, 5, 6, 7, 8));
- }
-
- @Test
- public void testBasics() {
- for (int i = 0; i < basicFilters.size(); i++) {
- assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
- }
- }
-
- @Test
- public void testLogicalAnd() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.retainAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testLogicalOr() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.addAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testNotEvaluable() {
- ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
- assertEquals(all, index.filter(notEvaluable));
-
- LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
- assertEquals(all, index.filter(or));
-
- LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
- assertEquals(basicResults.get(0), index.filter(and));
- }
-
- public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
- CompareTupleFilter filter = new CompareTupleFilter(op);
- filter.addChild(columnFilter(col));
- for (int i : ids) {
- filter.addChild(constFilter(i));
- }
- return filter;
- }
-
- public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
- LogicalTupleFilter filter = new LogicalTupleFilter(op);
- for (TupleFilter f : filters)
- filter.addChild(f);
- return filter;
- }
-
- public static ColumnTupleFilter columnFilter(TblColRef col) {
- return new ColumnTupleFilter(col);
- }
-
- public static ConstantTupleFilter constFilter(int id) {
- byte[] space = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(space);
- StringSerializer stringSerializer = new StringSerializer(DataType.getType("string"));
- stringSerializer.serialize("" + id, buf);
- ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
- return new ConstantTupleFilter(data);
- }
-
- public static ConciseSet set(int... ints) {
- ConciseSet set = new ConciseSet();
- for (int i : ints)
- set.add(i);
- return set;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 36d34c8..9478555 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -39,6 +39,15 @@
<artifactId>kylin-core-cube</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 87c4438..0832e80 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -39,6 +39,29 @@
<artifactId>kylin-core-common</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <!-- Compiled -->
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
index 12162b2..fbdb0bb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Map;
@@ -33,7 +34,7 @@ import com.google.common.collect.Maps;
/**
* Encoding Boolean values to bytes
*/
-public class BooleanDimEnc extends DimensionEncoding {
+public class BooleanDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
public static final String ENCODING_NAME = "boolean";
@@ -108,7 +109,7 @@ public class BooleanDimEnc extends DimensionEncoding {
return new BooleanSerializer();
}
- private class BooleanSerializer extends DataTypeSerializer<Object> {
+ public class BooleanSerializer extends DataTypeSerializer<Object> {
private byte[] currentBuf() {
byte[] buf = (byte[]) current.get();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
index 79e00ce..3b390d9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
@@ -18,10 +18,12 @@
package org.apache.kylin.dimension;
+import java.io.Serializable;
+
/**
* This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility.
*/
-public class DateDimEnc extends AbstractDateDimEnc {
+public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
private static final long serialVersionUID = 1L;
public static final int ID_9999_12_31 = 3652426;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index 8244152..46f9117 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
@@ -29,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class DictionaryDimEnc extends DimensionEncoding {
+public class DictionaryDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(DictionaryDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index f7f02a0..9ce1577 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -29,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class FixedLenDimEnc extends DimensionEncoding {
+public class FixedLenDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(FixedLenDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
index f90a40e..a931450 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -43,7 +44,7 @@ import com.google.common.base.Preconditions;
* <p>
* Due to these limitations hex representation of hash values(with no padding, better with even characters) is more suitable
*/
-public class FixedLenHexDimEnc extends DimensionEncoding {
+public class FixedLenHexDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(FixedLenHexDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
index aa954da..3650200 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory;
* deprecated use IntegerDimEnc instead
* @deprecated
*/
-public class IntDimEnc extends DimensionEncoding {
+public class IntDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(IntDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 0875a7f..e024696 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory;
* -2^(8*N-1) is not supported because the slot is reserved for null values.
* -2^(8*N-1) will be encoded with warn, and its output will be null
*/
-public class IntegerDimEnc extends DimensionEncoding {
+public class IntegerDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(IntegerDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
index c0c52d1..d998f44 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
@@ -26,13 +26,14 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
/**
* not being used yet, prepared for future
*/
-public class OneMoreByteVLongDimEnc extends DimensionEncoding {
+public class OneMoreByteVLongDimEnc extends DimensionEncoding implements Serializable{
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(OneMoreByteVLongDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
index 8c66d5d..4bef2b8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
@@ -18,10 +18,12 @@
package org.apache.kylin.dimension;
+import java.io.Serializable;
+
/**
* This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility.
*/
-public class TimeDimEnc extends AbstractDateDimEnc {
+public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
private static final long serialVersionUID = 1L;
public static final String ENCODING_NAME = "time";
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 5d0e007..7f3a5f1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -218,7 +218,7 @@ abstract public class MeasureTypeFactory<T> {
}
@SuppressWarnings("rawtypes")
- private static class NeedRewriteOnlyMeasureType extends MeasureType {
+ public static class NeedRewriteOnlyMeasureType extends MeasureType {
private Boolean needRewrite;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
index 438a33f..c635cd6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
@@ -77,7 +77,7 @@ public class HLLDistinctCountAggFunc {
}
@SuppressWarnings("serial")
- private static class FixedValueHLLCMockup extends HLLCounter {
+ public static class FixedValueHLLCMockup extends HLLCounter {
private Long value = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
index 2f625af..d8fdc6e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
@@ -22,13 +22,14 @@ import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.io.Serializable;
/**
* Modified from Counter.java in https://github.com/addthis/stream-lib
*
* @param <T>
*/
-public class Counter<T> implements Externalizable {
+public class Counter<T> implements Externalizable, Serializable{
protected T item;
protected double count;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 6b800bb..1c85f09 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -48,11 +49,11 @@ import com.google.common.collect.Sets;
public class DataModelDesc extends RootPersistentEntity {
private static final Logger logger = LoggerFactory.getLogger(DataModelDesc.class);
- public static enum TableKind {
+ public static enum TableKind implements Serializable{
FACT, LOOKUP
}
- public static enum RealizationCapacity {
+ public static enum RealizationCapacity implements Serializable{
SMALL, MEDIUM, LARGE
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 3ab9a46..c7666cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -113,7 +113,7 @@ public class JoinsTree implements Serializable {
return chain.join;
}
- static class Chain implements Serializable {
+ public static class Chain implements Serializable {
private static final long serialVersionUID = 1L;
TableRef table; // pk side
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index c6e6425..38c2de5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -34,7 +34,7 @@ import java.io.Serializable;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class PartitionDesc implements Serializable {
- public static enum PartitionType {
+ public static enum PartitionType implements Serializable{
APPEND, //
UPDATE_INSERT // not used since 0.7.1
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
index 65c44df..689a314 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
@@ -18,10 +18,12 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+
/**
* @author xduo
*
*/
-public enum SegmentStatusEnum {
+public enum SegmentStatusEnum implements Serializable{
NEW, READY, READY_PENDING
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 9371f76..9d22dc6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata.model;
import java.io.IOException;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -27,7 +28,7 @@ import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class Segments<T extends ISegment> extends ArrayList<T> {
+public class Segments<T extends ISegment> extends ArrayList<T> implements Serializable{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
index 27e2d57..872d2e4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
@@ -18,7 +18,9 @@
package org.apache.kylin.metadata.realization;
-public enum RealizationStatusEnum {
+import java.io.Serializable;
+
+public enum RealizationStatusEnum implements Serializable{
DISABLED, READY, DESCBROKEN
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 9bd3f04..8c3da5f 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -39,8 +39,28 @@
<artifactId>kylin-core-cube</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index be2cdf6..7a478f3 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -67,11 +67,6 @@
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
-
<!-- Hadoop dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -116,45 +111,13 @@
<artifactId>maven-model</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.10</version>
+ <scope>test</scope>
</dependency>
-
-
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <minimizeJar>false</minimizeJar>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>job</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-
</project>
\ No newline at end of file