You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/12/25 14:49:22 UTC
[6/6] kylin git commit: Use java built-in class to replace
LongMutable/DoubleMutable;
Use java built-in class to replace LongMutable/DoubleMutable;
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55ba39af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55ba39af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55ba39af
Branch: refs/heads/sparkcubing-rebase
Commit: 55ba39afb6ded85e5f8d82df2ba6828dc4b68819
Parents: 0a648fb
Author: shaofengshi <sh...@apache.org>
Authored: Sun Dec 25 22:48:43 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Dec 25 22:48:43 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 11 ++++++
.../apache/kylin/common/KylinConfigBase.java | 17 +++++++-
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +
.../apache/kylin/gridtable/UnitTestSupport.java | 41 ++++++++++----------
.../gridtable/AggregationCacheMemSizeTest.java | 6 +--
.../gridtable/AggregationCacheSpillTest.java | 5 +--
.../kylin/gridtable/SimpleGridTableTest.java | 11 +++---
.../gridtable/SimpleInvertedIndexTest.java | 2 +-
.../metadata/measure/MeasureCodecTest.java | 6 +--
.../kylin/measure/basic/DoubleIngester.java | 16 +++-----
.../measure/basic/DoubleMaxAggregator.java | 15 ++++---
.../measure/basic/DoubleMinAggregator.java | 15 ++++---
.../measure/basic/DoubleSumAggregator.java | 18 +++++----
.../kylin/measure/basic/LongIngester.java | 16 +++-----
.../kylin/measure/basic/LongMaxAggregator.java | 15 ++++---
.../kylin/measure/basic/LongMinAggregator.java | 15 ++++---
.../kylin/measure/basic/LongSumAggregator.java | 18 +++++----
.../metadata/datatype/BooleanSerializer.java | 29 ++++----------
.../metadata/datatype/DateTimeSerializer.java | 25 ++++--------
.../metadata/datatype/DoubleSerializer.java | 25 ++++--------
.../kylin/metadata/datatype/Int4Serializer.java | 25 ++++--------
.../metadata/datatype/Long8Serializer.java | 24 ++++--------
.../kylin/metadata/datatype/LongSerializer.java | 25 ++++--------
.../measure/AggregatorMemEstimateTest.java | 4 +-
.../storage/gtrecord/DictGridTableTest.java | 41 ++++++++++----------
.../kylin/engine/mr/steps/CubeReducerTest.java | 3 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 32 +++++----------
.../test_case_data/sandbox/kylin.properties | 6 +--
.../hbase/steps/RowValueDecoderTest.java | 7 ++--
29 files changed, 199 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 131a725..dabe113 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -128,6 +128,17 @@ kylin.engine.mr.max-reducer-number=500
kylin.engine.mr.mapper-input-rows=1000000
+### Spark Engine ###
+
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
+kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+
+# Spark install home, default be $KYLIN_HOME/spark/
+#kylin.engine.spark.spark-home=
+
+# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties
+#kylin.engine.spark.properties-file=
+
### CUBE | DICTIONARY ###
# 'auto', 'inmem', 'layer' or 'random' for testing
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c32550f..5bec9b3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -714,7 +714,20 @@ abstract public class KylinConfigBase implements Serializable {
// ============================================================================
public String getSparkHome() {
- return getRequired("kylin.engine.spark.spark-home");
+ String sparkHome = getOptional("kylin.engine.spark.spark-home", "spark");
+ File f = new File(sparkHome);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ } else {
+ String home = getKylinHome();
+ f = new File(home, sparkHome);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ }
+ }
+
+ throw new IllegalArgumentException("Spark home '" + sparkHome + "' does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties");
+
}
public String getSparkHadoopConfDir() {
@@ -734,7 +747,7 @@ abstract public class KylinConfigBase implements Serializable {
}
}
- throw new IllegalArgumentException("Spark conf file '" + conf + "' does not exist.");
+ throw new IllegalArgumentException("Spark conf properties file '" + conf + "' does not exist.");
}
public String getSparkAdditionalJars() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 66a444e..b4cba39 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -462,6 +462,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
for (int i = 0; i < totalSum.length; i++) {
if (totalSum[i] instanceof DoubleMutable) {
totalSum[i] = Math.round(((DoubleMutable) totalSum[i]).get());
+ } else if (totalSum[i] instanceof Double) {
+ totalSum[i] = Math.round(((Double) totalSum[i]).doubleValue());
} else if (totalSum[i] instanceof TopNCounter) {
TopNCounter counter = (TopNCounter) totalSum[i];
Iterator<Counter> iterator = counter.iterator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index b8d116c..6190f29 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -28,7 +28,6 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo.Builder;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
public class UnitTestSupport {
@@ -84,16 +83,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5")));
}
return result;
}
@@ -106,16 +105,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 8ffe055..cd292a2 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -27,8 +27,6 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
import org.junit.Test;
@@ -92,10 +90,10 @@ public class AggregationCacheMemSizeTest {
private MeasureAggregator<?>[] createNoMemHungryAggrs() {
LongSumAggregator longSum = new LongSumAggregator();
- longSum.aggregate(new LongMutable(10));
+ longSum.aggregate(new Long(10));
DoubleSumAggregator doubleSum = new DoubleSumAggregator();
- doubleSum.aggregate(new DoubleMutable(10));
+ doubleSum.aggregate(new Double(10));
BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index d7787e0..74c2659 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,7 +90,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
for (GTRecord record : scanner) {
assertNotNull(record);
Object[] returnRecord = record.getValues();
- assertEquals(20, ((LongMutable) returnRecord[3]).get());
+ assertEquals(20, ((Long) returnRecord[3]).longValue());
assertEquals(21, ((BigDecimal) returnRecord[4]).longValue());
count++;
@@ -133,7 +132,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
for (GTRecord record : scanner) {
assertNotNull(record);
Object[] returnRecord = record.getValues();
- assertEquals(80000, ((LongMutable) returnRecord[3]).get());
+ assertEquals(80000, ((Long) returnRecord[3]).longValue());
assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue());
count++;
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index fd571d0..5dd7214 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -96,7 +95,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
Object[] v = r.getValues();
assertTrue(((String) v[0]).startsWith("2015-"));
assertTrue(((String) v[2]).equals("Food"));
- assertTrue(((LongMutable) v[3]).get() == 10);
+ assertTrue(((Long) v[3]).longValue() == 10);
assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
System.out.println(r);
}
@@ -113,19 +112,19 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
Object[] v = r.getValues();
switch (i) {
case 0:
- assertTrue(((LongMutable) v[3]).get() == 20);
+ assertTrue(((Long) v[3]).longValue() == 20);
assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
break;
case 1:
- assertTrue(((LongMutable) v[3]).get() == 30);
+ assertTrue(((Long) v[3]).longValue() == 30);
assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
break;
case 2:
- assertTrue(((LongMutable) v[3]).get() == 40);
+ assertTrue(((Long) v[3]).longValue() == 40);
assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
break;
case 3:
- assertTrue(((LongMutable) v[3]).get() == 10);
+ assertTrue(((Long) v[3]).longValue() == 10);
assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
break;
default:
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 1026414..388a08b 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -71,7 +71,7 @@ public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
GTRowBlock.Writer writer = mockBlock.getWriter();
GTRecord record = new GTRecord(info);
for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new LongMutable(0), new BigDecimal(0));
+ record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
for (int j = 0; j < info.getRowBlockSize(); j++) {
writer.append(record);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 0f3f3a9..d2122d6 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -27,8 +27,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.AfterClass;
@@ -54,8 +52,8 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") };
BufferedMeasureCodec codec = new BufferedMeasureCodec(descs);
- DoubleMutable d = new DoubleMutable(1.0);
- LongMutable l = new LongMutable(2);
+ Double d = new Double(1.0);
+ Long l = new Long(2);
BigDecimal b = new BigDecimal("333.1234");
HLLCounter hllc = new HLLCounter(16);
hllc.add("1234567");
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index e42f275..90ecb0d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -22,30 +22,24 @@ import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class DoubleIngester extends MeasureIngester<DoubleMutable> {
-
- // avoid repeated object creation
- private DoubleMutable current = new DoubleMutable();
+public class DoubleIngester extends MeasureIngester<Double> {
@Override
- public DoubleMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public Double valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
- DoubleMutable l = current;
if (values[0] == null || values[0].length() == 0)
- l.set(0L);
+ return new Double(0);
else
- l.set(Double.parseDouble(values[0]));
- return l;
+ return Double.parseDouble(values[0]);
}
@Override
public void reset() {
- current = new DoubleMutable();
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
index 25911e8..af24390 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleMaxAggregator extends MeasureAggregator<Double> {
- DoubleMutable max = null;
+ Double max = null;
@Override
public void reset() {
@@ -34,15 +33,15 @@ public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
}
@Override
- public void aggregate(DoubleMutable value) {
+ public void aggregate(Double value) {
if (max == null)
- max = new DoubleMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
+ max = value;
+ else if (max < value)
+ max = value;
}
@Override
- public DoubleMutable getState() {
+ public Double getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
index be97deb..0ef4734 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleMinAggregator extends MeasureAggregator<Double> {
- DoubleMutable min = null;
+ Double min = null;
@Override
public void reset() {
@@ -34,15 +33,15 @@ public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
}
@Override
- public void aggregate(DoubleMutable value) {
+ public void aggregate(Double value) {
if (min == null)
- min = new DoubleMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
+ min = value;
+ else if (min > value)
+ min = value;
}
@Override
- public DoubleMutable getState() {
+ public Double getState() {
return min;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
index 29eb787..58f714e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
@@ -19,28 +19,30 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleSumAggregator extends MeasureAggregator<Double> {
- DoubleMutable sum = new DoubleMutable();
+ Double sum = null;
@Override
public void reset() {
-// sum.set(0.0);
- sum = new DoubleMutable();
+ sum = null;
}
@Override
- public void aggregate(DoubleMutable value) {
- sum.set(sum.get() + value.get());
+ public void aggregate(Double value) {
+ if (sum == null) {
+ sum = value;
+ } else {
+ sum = sum + value;
+ }
}
@Override
- public DoubleMutable getState() {
+ public Double getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index 439f096..f7382b9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -22,30 +22,24 @@ import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class LongIngester extends MeasureIngester<LongMutable> {
-
- // avoid repeated object creation
- private LongMutable current = new LongMutable();
+public class LongIngester extends MeasureIngester<Long> {
@Override
- public LongMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
- LongMutable l = current;
if (values[0] == null || values[0].length() == 0)
- l.set(0L);
+ return new Long(0L);
else
- l.set(Long.parseLong(values[0]));
- return l;
+ return Long.parseLong(values[0]);
}
@Override
public void reset() {
- current = new LongMutable();
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
index ca44f15..c7556e3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
+public class LongMaxAggregator extends MeasureAggregator<Long> {
- LongMutable max = null;
+ Long max = null;
@Override
public void reset() {
@@ -34,15 +33,15 @@ public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
}
@Override
- public void aggregate(LongMutable value) {
+ public void aggregate(Long value) {
if (max == null)
- max = new LongMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
+ max = value;
+ else if (max < value)
+ max = value;
}
@Override
- public LongMutable getState() {
+ public Long getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
index dadc64e..135ce10 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongMinAggregator extends MeasureAggregator<LongMutable> {
+public class LongMinAggregator extends MeasureAggregator<Long> {
- LongMutable min = null;
+ Long min = null;
@Override
public void reset() {
@@ -34,15 +33,15 @@ public class LongMinAggregator extends MeasureAggregator<LongMutable> {
}
@Override
- public void aggregate(LongMutable value) {
+ public void aggregate(Long value) {
if (min == null)
- min = new LongMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
+ min = value;
+ else if (min > value)
+ min = value;
}
@Override
- public LongMutable getState() {
+ public Long getState() {
return min;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
index 0a56af8..28d45c0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
@@ -19,28 +19,30 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongSumAggregator extends MeasureAggregator<LongMutable> {
+public class LongSumAggregator extends MeasureAggregator<Long> {
- LongMutable sum = new LongMutable();
+ Long sum = null;
@Override
public void reset() {
-// sum.set(0);
- sum = new LongMutable();
+ sum = null;
}
@Override
- public void aggregate(LongMutable value) {
- sum.set(sum.get() + value.get());
+ public void aggregate(Long value) {
+ if (sum == null) {
+ sum = value;
+ } else {
+ sum = sum + value;
+ }
}
@Override
- public LongMutable getState() {
+ public Long getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
index 998e5e9..3d485d2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
@@ -23,7 +23,7 @@ import org.apache.commons.lang.BooleanUtils;
import java.nio.ByteBuffer;
-public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
+public class BooleanSerializer extends DataTypeSerializer<Long> {
public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" };
@@ -31,24 +31,13 @@ public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = (LongMutable) current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ out.putLong(value);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return in.getLong();
}
@Override
@@ -67,12 +56,10 @@ public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- LongMutable l = current();
+ public Long valueOf(String str) {
if (str == null)
- l.set(0L);
+ return Long.valueOf(0L);
else
- l.set(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase())));
- return l;
+ return Long.valueOf(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase())));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
index a5719bd..5101766 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
@@ -22,30 +22,19 @@ import org.apache.kylin.common.util.DateFormat;
import java.nio.ByteBuffer;
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
+public class DateTimeSerializer extends DataTypeSerializer<Long> {
public DateTimeSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = (LongMutable) current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ out.putLong(value);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return in.getLong();
}
@Override
@@ -64,8 +53,8 @@ public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(DateFormat.stringToMillis(str));
+ public Long valueOf(String str) {
+ return Long.valueOf(DateFormat.stringToMillis(str));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
index cda4ff5..20cfff3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
@@ -22,30 +22,19 @@ import java.nio.ByteBuffer;
/**
*/
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
+public class DoubleSerializer extends DataTypeSerializer<Double> {
public DoubleSerializer(DataType type) {
}
@Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = (DoubleMutable) current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
+ public void serialize(Double value, ByteBuffer out) {
+ out.putDouble(value);
}
@Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
+ public Double deserialize(ByteBuffer in) {
+ return in.getDouble();
}
@Override
@@ -64,7 +53,7 @@ public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
}
@Override
- public DoubleMutable valueOf(String str) {
- return new DoubleMutable(Double.parseDouble(str));
+ public Double valueOf(String str) {
+ return Double.parseDouble(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
index c726ab6..0e82e11 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
@@ -24,30 +24,19 @@ import java.nio.ByteBuffer;
/**
*/
-public class Int4Serializer extends DataTypeSerializer<IntMutable> {
+public class Int4Serializer extends DataTypeSerializer<Integer> {
public Int4Serializer(DataType type) {
}
@Override
- public void serialize(IntMutable value, ByteBuffer out) {
- BytesUtil.writeUnsigned(value.get(), 4, out);
- }
-
- private IntMutable current() {
- IntMutable l = (IntMutable) current.get();
- if (l == null) {
- l = new IntMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Integer value, ByteBuffer out) {
+ BytesUtil.writeUnsigned(value, 4, out);
}
@Override
- public IntMutable deserialize(ByteBuffer in) {
- IntMutable l = current();
- l.set(BytesUtil.readUnsigned(in, 4));
- return l;
+ public Integer deserialize(ByteBuffer in) {
+ return BytesUtil.readUnsigned(in, 4);
}
@Override
@@ -66,7 +55,7 @@ public class Int4Serializer extends DataTypeSerializer<IntMutable> {
}
@Override
- public IntMutable valueOf(String str) {
- return new IntMutable(Integer.parseInt(str));
+ public Integer valueOf(String str) {
+ return Integer.parseInt(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
index 186cdb9..7dd5aa7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
@@ -24,30 +24,20 @@ import java.nio.ByteBuffer;
/**
*/
-public class Long8Serializer extends DataTypeSerializer<LongMutable> {
+public class Long8Serializer extends DataTypeSerializer<Long> {
public Long8Serializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeLong(value.get(), out);
+ public void serialize(Long value, ByteBuffer out) {
+ BytesUtil.writeLong(value, out);
}
- private LongMutable current() {
- LongMutable l = (LongMutable) current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readLong(in));
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return BytesUtil.readLong(in);
}
@Override
@@ -66,7 +56,7 @@ public class Long8Serializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(Long.parseLong(str));
+ public Long valueOf(String str) {
+ return Long.parseLong(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
index d8f3f37..605dcd7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
@@ -24,30 +24,19 @@ import java.nio.ByteBuffer;
/**
*/
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
+public class LongSerializer extends DataTypeSerializer<Long> {
public LongSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = (LongMutable) current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ BytesUtil.writeVLong(value, out);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return BytesUtil.readVLong(in);
}
@Override
@@ -72,7 +61,7 @@ public class LongSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(Long.parseLong(str));
+ public Long valueOf(String str) {
+ return Long.parseLong(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 0f22610..af70196 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -52,7 +52,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
}
private List<? extends MeasureAggregator> basicAggregators() {
- LongMutable longVal = new LongMutable(1000);
+ Long longVal = new Long(1000);
LongMinAggregator longMin = new LongMinAggregator();
LongMaxAggregator longMax = new LongMaxAggregator();
LongSumAggregator longSum = new LongSumAggregator();
@@ -60,7 +60,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
longMax.aggregate(longVal);
longSum.aggregate(longVal);
- DoubleMutable doubleVal = new DoubleMutable(1.0);
+ Double doubleVal = new Double(1.0);
DoubleMinAggregator doubleMin = new DoubleMinAggregator();
DoubleMaxAggregator doubleMax = new DoubleMaxAggregator();
DoubleSumAggregator doubleSum = new DoubleSumAggregator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index bac9c10..b90e3a1 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -52,7 +52,6 @@ import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -499,16 +498,16 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
GTRecord r = new GTRecord(table.getInfo());
GTBuilder builder = table.rebuild();
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5")));
builder.close();
return table;
@@ -524,34 +523,34 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
for (int i = 0; i < 100000; i++) {
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5")));
}
builder.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 97dd750..29ca9b8 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -41,7 +41,6 @@ import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
@@ -161,7 +160,7 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
}
private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongMutable(count), new LongMutable(item_count) };
+ Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) };
ByteBuffer buf = codec.encode(values);
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 7467e86..8615a13 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -71,8 +71,6 @@ import scala.Tuple2;
import javax.annotation.Nullable;
import java.io.File;
import java.io.FileFilter;
-import java.io.IOException;
-import java.io.ObjectInputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -309,31 +307,24 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
BufferedMeasureCodec codec;
CubeDesc cubeDesc;
int measureNum;
- transient ThreadLocal<MeasureAggregators> current = new ThreadLocal<>();
+ MeasureAggregators aggregators;
CuboidReducerFunction2(int measureNum, CubeDesc cubeDesc, BufferedMeasureCodec codec) {
this.codec = codec;
this.cubeDesc = cubeDesc;
this.measureNum = measureNum;
+ this.aggregators = new MeasureAggregators(cubeDesc.getMeasures());
}
@Override
public Object[] call(Object[] input1, Object[] input2) throws Exception {
- if (current.get() == null) {
- current.set(new MeasureAggregators(cubeDesc.getMeasures()));
- }
Object[] result = new Object[measureNum];
- current.get().reset();
- current.get().aggregate(input1);
- current.get().aggregate(input2);
- current.get().collectStates(result);
+ aggregators.reset();
+ aggregators.aggregate(input1);
+ aggregators.aggregate(input2);
+ aggregators.collectStates(result);
return result;
}
-
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- in.defaultReadObject();
- current = new ThreadLocal();
- }
}
class BaseCuboidReducerFunction2 extends CuboidReducerFunction2 {
@@ -346,14 +337,11 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
@Override
public Object[] call(Object[] input1, Object[] input2) throws Exception {
- if (current.get() == null) {
- current.set(new MeasureAggregators(cubeDesc.getMeasures()));
- }
- current.get().reset();
Object[] result = new Object[measureNum];
- current.get().aggregate(input1, needAggr);
- current.get().aggregate(input2, needAggr);
- current.get().collectStates(result);
+ aggregators.reset();
+ aggregators.aggregate(input1, needAggr);
+ aggregators.aggregate(input2, needAggr);
+ aggregators.collectStates(result);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index db8eb7a..74a6a8a 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -157,9 +157,7 @@ kylin.env=DEV
kylin.source.hive.keep-flat-table=true
### Spark ###
-#kylin.engine.spark.spark-home=/usr/hdp/2.2.4.2-2/spark
#kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
-kylin.engine.spark.env.hadoop-conf-dir=/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox
+kylin.engine.spark.env.hadoop-conf-dir=../examples/test_case_data/sandbox
kylin.engine.spark.spark-home=/Users/shishaofeng/spark-1.6.3-bin-hadoop2.6
-kylin.engine.spark.properties-file=/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/kylin-spark-conf.properties
-kylin.engine.spark.conf.jars=
\ No newline at end of file
+kylin.engine.spark.properties-file=../examples/test_case_data/sandbox/kylin-spark-conf.properties
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index f0ad6e0..9b1a00d 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -31,7 +31,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
@@ -60,8 +59,8 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
BigDecimal sum = new BigDecimal("333.1234567");
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
- LongMutable count = new LongMutable(2);
- LongMutable item_count = new LongMutable(100);
+ Long count = new Long(2);
+ Long item_count = new Long(100);
ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });
buf.flip();
@@ -91,7 +90,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
LongWritable count = new LongWritable(2);
- LongMutable item_count = new LongMutable(100);
+ Long item_count = new Long(100);
codec.encode(new Object[] { sum, min, max, count, item_count });
}