You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/06/05 05:23:10 UTC
[22/67] [abbrv] kylin git commit: Revert "reformat code"
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 2747ca0..f749fb4 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -93,10 +93,10 @@ public class AggregationCacheMemSizeTest {
}
enum Settings {
- WITHOUT_MEM_HUNGRY, // only test basic aggrs
- WITH_HLLC, // basic aggrs + hllc
- WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap
- WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap
+ WITHOUT_MEM_HUNGRY, // only test basic aggrs
+ WITH_HLLC, // basic aggrs + hllc
+ WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap
+ WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap
}
private MeasureAggregator<?>[] createNoMemHungryAggrs() {
@@ -132,19 +132,19 @@ public class AggregationCacheMemSizeTest {
aggregators.addAll(Arrays.asList(createNoMemHungryAggrs()));
switch (settings) {
- case WITHOUT_MEM_HUNGRY:
- break;
- case WITH_HLLC:
- aggregators.add(createHLLCAggr());
- break;
- case WITH_LOW_CARD_BITMAP:
- aggregators.add(createBitmapAggr(true));
- break;
- case WITH_HIGH_CARD_BITMAP:
- aggregators.add(createBitmapAggr(false));
- break;
- default:
- break;
+ case WITHOUT_MEM_HUNGRY:
+ break;
+ case WITH_HLLC:
+ aggregators.add(createHLLCAggr());
+ break;
+ case WITH_LOW_CARD_BITMAP:
+ aggregators.add(createBitmapAggr(true));
+ break;
+ case WITH_HIGH_CARD_BITMAP:
+ aggregators.add(createBitmapAggr(false));
+ break;
+ default:
+ break;
}
return aggregators.toArray(new MeasureAggregator[aggregators.size()]);
@@ -160,8 +160,8 @@ public class AggregationCacheMemSizeTest {
System.out.printf("%-15s %-10s %-10s\n", "cardinality", "estimate", "actual");
for (BitmapAggregator aggr : bitmapAggrs) {
- System.out.printf("%-15d %-10d %-10d\n", aggr.getState().getCount(), aggr.getMemBytesEstimate(),
- meter.measureDeep(aggr));
+ System.out.printf("%-15d %-10d %-10d\n",
+ aggr.getState().getCount(), aggr.getMemBytesEstimate(), meter.measureDeep(aggr));
}
}
@@ -190,8 +190,8 @@ public class AggregationCacheMemSizeTest {
long actualMillis = 0;
System.out.println("Settings: " + settings);
- System.out.printf("%15s %15s %15s %15s %15s\n", "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)",
- "Actual(ms)");
+ System.out.printf("%15s %15s %15s %15s %15s\n",
+ "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)", "Actual(ms)");
for (int i = 0; i < inputCount; i++) {
byte[] key = new byte[10];
@@ -199,7 +199,7 @@ public class AggregationCacheMemSizeTest {
MeasureAggregator[] values = createAggrs(settings);
map.put(key, values);
- if ((i + 1) % reportInterval == 0) {
+ if ((i+1) % reportInterval == 0) {
stopwatch.start();
long estimateBytes = GTAggregateScanner.estimateSizeOfAggrCache(key, values, map.size());
estimateMillis += stopwatch.elapsedMillis();
@@ -210,8 +210,8 @@ public class AggregationCacheMemSizeTest {
actualMillis += stopwatch.elapsedMillis();
stopwatch.reset();
- System.out.printf("%,15d %,15d %,15d %,15d %,15d\n", map.size(), estimateBytes, actualBytes,
- estimateMillis, actualMillis);
+ System.out.printf("%,15d %,15d %,15d %,15d %,15d\n",
+ map.size(), estimateBytes, actualBytes, estimateMillis, actualMillis);
}
}
System.out.println("---------------------------------------\n");
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index d9241e5..8b2243c 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -77,11 +77,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
};
- GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null)
- .setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3))
- .setAggrMetrics(new ImmutableBitSet(3, 6))
- .setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null)
- .setAggCacheMemThreshold(0.5).createGTScanRequest();
+ GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest);
@@ -118,11 +114,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
};
// all-in-mem testcase
- GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null)
- .setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3))
- .setAggrMetrics(new ImmutableBitSet(3, 6))
- .setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null)
- .setAggCacheMemThreshold(0.5).createGTScanRequest();
+ GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 770f6ce..1866079 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -45,13 +45,9 @@ public class DimEncodingPreserveOrderTest {
successValue.add(new long[] { -32767, -127, 0, 127, 32767 });
successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 8388607 });
successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L });
- successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607,
- 2147483647L, 549755813887L });
- successValue.add(new long[] { -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127,
- 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L });
- successValue.add(new long[] { -36028797018963967L, -140737488355327L, -549755813887L, -2147483647L, -8388607,
- -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L,
- 36028797018963967L });
+ successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L });
+ successValue.add(new long[] { -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L });
+ successValue.add(new long[] { -36028797018963967L, -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L, 36028797018963967L });
successValue.add(new long[] { //
-9223372036854775807L, //
-36028797018963967L, //
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
index aadcf45..1ae229a 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
@@ -88,8 +88,7 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
@Test
public void testGTInfo() {
- CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getCube("test_kylin_cube_with_slr_ready");
+ CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
CubeSegment segment = cube.getFirstSegment();
Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index 58bb76c..14a25c5 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -85,8 +85,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
}
private IGTScanner scan(GridTable table) throws IOException {
- GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null)
- .setFilterPushDown(null).createGTScanRequest();
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest();
IGTScanner scanner = table.scan(req);
for (GTRecord r : scanner) {
Object[] v = r.getValues();
@@ -101,9 +100,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
}
private IGTScanner scanAndAggregate(GridTable table) throws IOException {
- GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null)
- .setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4))
- .setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest();
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest();
IGTScanner scanner = table.scan(req);
int i = 0;
for (GTRecord r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index a11b84a..7129a5e 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -50,8 +50,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
@Test
public void basicTest() {
- MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"),
- measure("HLLC16"), measure("bitmap") };
+ MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") };
BufferedMeasureCodec codec = new BufferedMeasureCodec(descs);
Double d = new Double(1.0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
index bda2bf0..f04c742 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
@@ -51,8 +51,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
@Test
public void test() {
- CubeDesc desc = CubeDescManager.getInstance(getTestConfig())
- .getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+ CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
MeasureDesc topSellerMeasure = null;
@@ -62,8 +61,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
break;
}
}
- TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(
- topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType());
+ TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType());
topSellerMeasure.getFunction().getConfiguration().clear();
List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
@@ -71,8 +69,7 @@ public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);
TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
- topSellerMeasure.getFunction().getConfiguration()
- .put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6");
+ topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6");
colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
assertTrue(colsNeedDict.size() == 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 9df0e26..b9f0d2b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -17,16 +17,11 @@
*/
package org.apache.kylin.dict;
-import static com.google.common.base.Preconditions.checkState;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
@@ -37,12 +32,14 @@ import org.apache.kylin.dict.global.GlobalDictMetadata;
import org.apache.kylin.dict.global.GlobalDictStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import static com.google.common.base.Preconditions.checkState;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[] to
@@ -63,8 +60,7 @@ import com.google.common.cache.RemovalNotification;
*/
@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
public class AppendTrieDictionary<T> extends CacheDictionary<T> {
- public static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65,
- 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict"
+ public static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict"
public static final int HEAD_SIZE_I = HEAD_MAGIC.length;
private static final Logger logger = LoggerFactory.getLogger(AppendTrieDictionary.class);
@@ -81,23 +77,19 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
final Path latestVersionPath = globalDictStore.getVersionDir(latestVersion);
this.metadata = globalDictStore.getMetadata(latestVersion);
this.bytesConvert = metadata.bytesConverter;
- this.dictCache = CacheBuilder.newBuilder().softValues()
- .removalListener(new RemovalListener<AppendDictSliceKey, AppendDictSlice>() {
- @Override
- public void onRemoval(RemovalNotification<AppendDictSliceKey, AppendDictSlice> notification) {
- logger.info("Evict slice with key {} and value {} caused by {}, size {}/{}",
- notification.getKey(), notification.getValue(), notification.getCause(),
- dictCache.size(), metadata.sliceFileMap.size());
- }
- }).build(new CacheLoader<AppendDictSliceKey, AppendDictSlice>() {
- @Override
- public AppendDictSlice load(AppendDictSliceKey key) throws Exception {
- AppendDictSlice slice = globalDictStore.readSlice(latestVersionPath.toString(),
- metadata.sliceFileMap.get(key));
- logger.info("Load slice with key {} and value {}", key, slice);
- return slice;
- }
- });
+ this.dictCache = CacheBuilder.newBuilder().softValues().removalListener(new RemovalListener<AppendDictSliceKey, AppendDictSlice>() {
+ @Override
+ public void onRemoval(RemovalNotification<AppendDictSliceKey, AppendDictSlice> notification) {
+ logger.info("Evict slice with key {} and value {} caused by {}, size {}/{}", notification.getKey(), notification.getValue(), notification.getCause(), dictCache.size(), metadata.sliceFileMap.size());
+ }
+ }).build(new CacheLoader<AppendDictSliceKey, AppendDictSlice>() {
+ @Override
+ public AppendDictSlice load(AppendDictSliceKey key) throws Exception {
+ AppendDictSlice slice = globalDictStore.readSlice(latestVersionPath.toString(), metadata.sliceFileMap.get(key));
+ logger.info("Load slice with key {} and value {}", key, slice);
+ return slice;
+ }
+ });
}
@Override
@@ -191,4 +183,4 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
public boolean contains(Dictionary other) {
return false;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 9eafc43..7a48a1c 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -70,8 +70,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
}
} else if (tupleFilter instanceof LogicalTupleFilter) {
@SuppressWarnings("unchecked")
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren()
- .listIterator();
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
while (childIterator.hasNext()) {
TupleFilter transformed = transform(childIterator.next());
if (transformed != null) {
@@ -93,8 +92,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
if (dict == null)
return null;
- CompareTupleFilter translated = new CompareTupleFilter(
- builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
+ CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
translated.addChild(new ColumnTupleFilter(columnRef));
try {
@@ -120,12 +118,10 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
@SuppressWarnings({ "unchecked", "rawtypes" })
private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
- if (compTupleFilter.getFunction() == null
- || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)))
+ if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)))
return null;
- BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter
- .getFunction();
+ BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter.getFunction();
if (!builtInFunctionTupleFilter.isValid())
return null;
@@ -135,8 +131,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
if (dict == null)
return null;
- CompareTupleFilter translated = new CompareTupleFilter(
- builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
+ CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
translated.addChild(new ColumnTupleFilter(columnRef));
try {
@@ -147,8 +142,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
Class clazz = Primitives.wrap(computedVal.getClass());
Object targetVal = compTupleFilter.getFirstValue();
if (Primitives.isWrapperType(clazz))
- targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null,
- compTupleFilter.getFirstValue()));
+ targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
int comp = ((Comparable) computedVal).compareTo(targetVal);
boolean compResult = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java
index 798c9c9..74d5ec5 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java
@@ -18,10 +18,10 @@
package org.apache.kylin.dict;
-import java.util.Comparator;
-
import org.apache.kylin.common.util.ByteArray;
+import java.util.Comparator;
+
/**
* Created by xiefan on 16-10-28.
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index 15e28a6..156971d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -18,12 +18,12 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.util.Dictionary;
+
import java.lang.ref.SoftReference;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.kylin.common.util.Dictionary;
-
/**
*/
public abstract class CacheDictionary<T> extends Dictionary<T> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index f772caf..56b4994 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -83,13 +83,13 @@ public class DateStrDictionary extends Dictionary<String> {
return pattern.length();
}
+
@Override
final protected int getIdFromValueImpl(String value, int roundFlag) {
Date date = stringToDate(value, pattern);
int id = calcIdFromSeqNo((int) DateDimEnc.getNumOfDaysSince0000FromMillis(date.getTime()));
if (id < baseId || id > maxId)
- throw new IllegalArgumentException(
- "'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]");
+ throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]");
return id;
}
@@ -123,6 +123,7 @@ public class DateStrDictionary extends Dictionary<String> {
init(pattern, baseId);
}
+
@Override
public int hashCode() {
return 31 * baseId + pattern.hashCode();
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 14faff4..61a0664 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -60,13 +60,11 @@ public class DictionaryGenerator {
return builder;
}
- public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator)
- throws IOException {
+ public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
return buildDictionary(newDictionaryBuilder(dataType), null, valueEnumerator);
}
- static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo,
- IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) throws IOException {
int baseId = 0; // always 0 for now
int nSamples = 5;
ArrayList<String> samples = new ArrayList<String>(nSamples);
@@ -121,7 +119,7 @@ public class DictionaryGenerator {
public boolean addValue(String value) {
if (StringUtils.isBlank(value)) // empty string is treated as null
return false;
-
+
// detect date pattern on the first value
if (datePattern == null) {
for (String p : DATE_PATTERNS) {
@@ -136,7 +134,7 @@ public class DictionaryGenerator {
if (datePattern == null)
throw new IllegalArgumentException("Unknown date pattern for input value: " + value);
}
-
+
// check the date format
DateFormat.stringToDate(value, datePattern);
return true;
@@ -176,28 +174,28 @@ public class DictionaryGenerator {
private static class StringTrieDictBuilder implements IDictionaryBuilder {
int baseId;
TrieDictionaryBuilder builder;
-
+
@Override
public void init(DictionaryInfo info, int baseId) throws IOException {
this.baseId = baseId;
this.builder = new TrieDictionaryBuilder(new StringBytesConverter());
}
-
+
@Override
public boolean addValue(String value) {
if (value == null)
return false;
-
+
builder.addValue(value);
return true;
}
-
+
@Override
public Dictionary<String> build() throws IOException {
return builder.build(baseId);
}
}
-
+
private static class StringTrieDictForestBuilder implements IDictionaryBuilder {
TrieDictionaryForestBuilder builder;
@@ -225,28 +223,28 @@ public class DictionaryGenerator {
private static class NumberTrieDictBuilder implements IDictionaryBuilder {
int baseId;
NumberDictionaryBuilder builder;
-
+
@Override
public void init(DictionaryInfo info, int baseId) throws IOException {
this.baseId = baseId;
this.builder = new NumberDictionaryBuilder();
}
-
+
@Override
public boolean addValue(String value) {
if (StringUtils.isBlank(value)) // empty string is treated as null
return false;
-
+
builder.addValue(value);
return true;
}
-
+
@Override
public Dictionary<String> build() throws IOException {
return builder.build(baseId);
}
}
-
+
private static class NumberTrieDictForestBuilder implements IDictionaryBuilder {
NumberDictionaryForestBuilder builder;
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 6d27d98..ae5c0f1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -55,9 +55,8 @@ public class DictionaryInfo extends RootPersistentEntity {
public DictionaryInfo(ColumnDesc col, String dataType, TableSignature input) {
this(col.getTable().getIdentity(), col.getName(), col.getZeroBasedIndex(), dataType, input);
}
-
- public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType,
- TableSignature input) {
+
+ public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input) {
this.updateRandomUuid();
@@ -94,10 +93,7 @@ public class DictionaryInfo extends RootPersistentEntity {
// to decide if two dictionaries are built on the same table/column,
// regardless of their signature
public boolean isDictOnSameColumn(DictionaryInfo other) {
- return this.sourceTable.equalsIgnoreCase(other.sourceTable)
- && this.sourceColumn.equalsIgnoreCase(other.sourceColumn)
- && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType)
- && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass);
+ return this.sourceTable.equalsIgnoreCase(other.sourceTable) && this.sourceColumn.equalsIgnoreCase(other.sourceColumn) && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType) && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass);
}
public String getSourceTable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index aa84f3a..1628f4e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -95,8 +95,7 @@ public class DictionaryManager {
.removalListener(new RemovalListener<String, DictionaryInfo>() {
@Override
public void onRemoval(RemovalNotification<String, DictionaryInfo> notification) {
- DictionaryManager.logger.info("Dict with resource path " + notification.getKey()
- + " is removed due to " + notification.getCause());
+ DictionaryManager.logger.info("Dict with resource path " + notification.getKey() + " is removed due to " + notification.getCause());
}
})//
.maximumSize(config.getCachedDictMaxEntrySize())//
@@ -156,8 +155,7 @@ public class DictionaryManager {
largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath());
Dictionary<String> largestDictObject = largestDictInfo.getDictionaryObject();
if (largestDictObject.contains(newDict)) {
- logger.info("dictionary content " + newDict + ", is contained by dictionary at "
- + largestDictInfo.getResourcePath());
+ logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDictInfo.getResourcePath());
return largestDictInfo;
} else if (newDict.contains(largestDictObject)) {
logger.info("dictionary content " + newDict + " is by far the largest, save it");
@@ -233,8 +231,7 @@ public class DictionaryManager {
} else {
if (!firstDictInfo.isDictOnSameColumn(info)) {
// don't throw exception, just output warning as legacy cube segment may build dict on PK
- logger.warn("Merging dictionaries are not structurally equal : " + firstDictInfo.getResourcePath()
- + " and " + info.getResourcePath());
+ logger.warn("Merging dictionaries are not structurally equal : " + firstDictInfo.getResourcePath() + " and " + info.getResourcePath());
}
}
totalSize += info.getInput().getSize();
@@ -270,19 +267,16 @@ public class DictionaryManager {
logger.info("Use one of the merging dictionaries directly");
return dicts.get(0);
} else {
- Dictionary<String> newDict = DictionaryGenerator
- .mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts);
+ Dictionary<String> newDict = DictionaryGenerator.mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts);
return trySaveNewDict(newDict, newDictInfo);
}
}
- public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable)
- throws IOException {
+ public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable) throws IOException {
return buildDictionary(model, col, inpTable, null);
}
- public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable,
- String builderClass) throws IOException {
+ public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable, String builderClass) throws IOException {
if (inpTable.exists() == false)
return null;
@@ -291,8 +285,7 @@ public class DictionaryManager {
DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
String dupInfo = checkDupByInfo(dictInfo);
if (dupInfo != null) {
- logger.info(
- "Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
return getDictionaryInfo(dupInfo);
}
@@ -303,16 +296,13 @@ public class DictionaryManager {
return trySaveNewDict(dictionary, dictInfo);
}
- private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo,
- String builderClass, TblColRef col) throws IOException {
+ private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo, String builderClass, TblColRef col) throws IOException {
Dictionary<String> dictionary;
IDictionaryValueEnumerator columnValueEnumerator = null;
try {
- columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(),
- dictInfo.getSourceColumnIndex());
+ columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), dictInfo.getSourceColumnIndex());
if (builderClass == null) {
- dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()),
- columnValueEnumerator);
+ dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()), columnValueEnumerator);
} else {
IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator);
@@ -326,21 +316,18 @@ public class DictionaryManager {
return dictionary;
}
- public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable,
- Dictionary<String> dictionary) throws IOException {
+ public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, IReadableTable inpTable, Dictionary<String> dictionary) throws IOException {
DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
String dupInfo = checkDupByInfo(dictInfo);
if (dupInfo != null) {
- logger.info(
- "Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
return getDictionaryInfo(dupInfo);
}
return trySaveNewDict(dictionary, dictInfo);
}
- private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, IReadableTable inpTable)
- throws IOException {
+ private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, IReadableTable inpTable) throws IOException {
TblColRef srcCol = decideSourceData(model, col);
TableSignature inputSig = inpTable.getSignature();
if (inputSig == null) // table does not exists
@@ -387,8 +374,7 @@ public class DictionaryManager {
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
final ResourceStore store = MetadataManager.getInstance(config).getStore();
- final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class,
- DictionaryInfoSerializer.INFO_SERIALIZER);
+ final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
TableSignature input = dictInfo.getInput();
@@ -402,8 +388,7 @@ public class DictionaryManager {
private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException {
final ResourceStore store = MetadataManager.getInstance(config).getStore();
- final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class,
- DictionaryInfoSerializer.INFO_SERIALIZER);
+ final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
DictionaryInfo largestDict = null;
for (DictionaryInfo dictionaryInfo : allResources) {
@@ -459,10 +444,8 @@ public class DictionaryManager {
DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
- logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:"
- + loadDictObj + ") at " + resourcePath);
- DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class,
- loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER);
+ logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" + loadDictObj + ") at " + resourcePath);
+ DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER);
return info;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index 002a89a..a593371 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
@@ -63,8 +63,7 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder {
if (lock.lock(getLockPath(sourceColumn))) {
logger.info("processed {} values for {}", counter, sourceColumn);
} else {
- throw new RuntimeException(
- "Failed to create global dictionary on " + sourceColumn + " This client doesn't keep the lock");
+ throw new RuntimeException("Failed to create global dictionary on " + sourceColumn + " This client doesn't keep the lock");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
index 72491a3..0934a7d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
@@ -29,10 +29,10 @@ public interface IDictionaryBuilder {
/** Sets the dictionary info for the dictionary being built. Mainly for GlobalDictionaryBuilder. */
void init(DictionaryInfo info, int baseId) throws IOException;
-
+
/** Add a new value into dictionary, returns it is accepted (not null) or not. */
boolean addValue(String value);
-
+
/** Build the dictionary */
Dictionary<String> build() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
index cbd9108..f193d4f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
@@ -24,7 +24,7 @@ import java.io.IOException;
* Created by dongli on 10/28/15.
*/
public interface IDictionaryValueEnumerator {
-
+
String current() throws IOException;
boolean moveNext() throws IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
index 027b3dc..5c80a26 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
@@ -31,7 +31,7 @@ public class IterableDictionaryValueEnumerator implements IDictionaryValueEnumer
public IterableDictionaryValueEnumerator(String... strs) {
this(Arrays.asList(strs));
}
-
+
public IterableDictionaryValueEnumerator(Iterable<String> list) {
iterator = list.iterator();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index 33ed607..c1bc5d5 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -57,7 +57,7 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
while (curDictIndex < dictionaryList.size()) {
if (curKey <= curDict.getMaxId()) {
curValue = curDict.getValueFromId(curKey);
- curKey++;
+ curKey ++;
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
index 4df0c84..397ca9f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
@@ -46,7 +46,7 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
return codec;
}
- public Number2BytesConverter() {
+ public Number2BytesConverter(){
this.maxDigitsBeforeDecimalPoint = MAX_DIGITS_BEFORE_DECIMAL_POINT;
}
@@ -74,11 +74,11 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
if (badBegin) {
v = new BigDecimal(v).toPlainString();
}
-
+
while (v.contains(".") && (v.endsWith("0") || v.endsWith("."))) {
v = v.substring(0, v.length() - 1);
}
-
+
return v;
}
@@ -117,9 +117,9 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
return;
}
+
if (len > buf.length) {
- throw new IllegalArgumentException("Too many digits for NumberDictionary: "
- + Bytes.toString(value, offset, len) + ". Internal buffer is only " + buf.length + " bytes");
+ throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Internal buffer is only " + buf.length + " bytes");
}
boolean negative = value[offset] == '-';
@@ -149,9 +149,7 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
// prepend '0'
int nZeroPadding = maxDigitsBeforeDecimalPoint - (decimalPoint - start);
if (nZeroPadding < 0 || nZeroPadding + 1 > start)
- throw new IllegalArgumentException(
- "Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Expect "
- + maxDigitsBeforeDecimalPoint + " digits before decimal point at max.");
+ throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Expect " + maxDigitsBeforeDecimalPoint + " digits before decimal point at max.");
for (int i = 0; i < nZeroPadding; i++) {
buf[--start] = '0';
}
@@ -243,4 +241,4 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
return out - offset;
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index 89b4701..1377e8e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -18,6 +18,7 @@
package org.apache.kylin.dict;
+
import org.apache.kylin.common.util.ClassUtil;
/**
@@ -28,6 +29,7 @@ import org.apache.kylin.common.util.ClassUtil;
@Deprecated
public class NumberDictionary<T> extends TrieDictionary<T> {
+
// ============================================================================
public NumberDictionary() { // default constructor for Writable interface
@@ -48,8 +50,8 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
protected void setConverterByName(String converterName) throws Exception {
converterName = "org.apache.kylin.dict.Number2BytesConverter";
this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
- ((Number2BytesConverter) this.bytesConvert)
- .setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY);
+ ((Number2BytesConverter)this.bytesConvert).setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY);
}
-}
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index f7edf88..26e4f89 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -18,6 +18,7 @@
package org.apache.kylin.dict;
+
/**
* Use <code>NumberDictionaryForestBuilder</code> instead.
*
@@ -26,10 +27,12 @@ package org.apache.kylin.dict;
@Deprecated
public class NumberDictionaryBuilder extends TrieDictionaryBuilder<String> {
+
public NumberDictionaryBuilder() {
super(new Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT));
}
+
public NumberDictionary build(int baseId) {
byte[] trieBytes = buildTrieBytes(baseId);
NumberDictionary2 r = new NumberDictionary2(trieBytes);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
index afd013b..380cd1d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
@@ -41,12 +41,10 @@ public class NumberDictionaryForestBuilder extends TrieDictionaryForestBuilder<S
}
public NumberDictionaryForestBuilder(int baseId) {
- super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT),
- 0);
+ super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), 0);
}
public NumberDictionaryForestBuilder(int baseId, int maxTrieSizeMB) {
- super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), 0,
- maxTrieSizeMB);
+ super(new org.apache.kylin.dict.Number2BytesConverter(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT), 0, maxTrieSizeMB);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
index 960681a..7caf686 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
@@ -47,8 +47,7 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator {
} else {
// normal case
if (split.length <= colIndex) {
- throw new ArrayIndexOutOfBoundsException(
- "Column no. " + colIndex + " not found, line split is " + Arrays.asList(split));
+ throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split));
}
colStrValue = split[colIndex];
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java
index ff4cc64..8f9f74f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java
@@ -41,8 +41,7 @@ public class TableColumnValueSortedEnumerator implements IDictionaryValueEnumera
private PriorityQueue<ReaderBuffer> pq;
- public TableColumnValueSortedEnumerator(Collection<IReadableTable.TableReader> readers, int colIndex,
- final Comparator<String> comparator) {
+ public TableColumnValueSortedEnumerator(Collection<IReadableTable.TableReader> readers, int colIndex, final Comparator<String> comparator) {
this.readers = readers;
this.colIndex = colIndex;
this.comparator = comparator;
@@ -90,6 +89,7 @@ public class TableColumnValueSortedEnumerator implements IDictionaryValueEnumera
return false;
}
+
@Override
public void close() throws IOException {
for (IReadableTable.TableReader reader : readers) {
@@ -136,8 +136,7 @@ public class TableColumnValueSortedEnumerator implements IDictionaryValueEnumera
} else {
// normal case
if (split.length <= colIndex) {
- throw new ArrayIndexOutOfBoundsException(
- "Column no. " + colIndex + " not found, line split is " + Arrays.asList(split));
+ throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split));
}
this.cache = split[colIndex];
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 160bfc5..8849015 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -29,6 +29,7 @@ import java.io.ObjectOutputStream;
import java.io.PrintStream;
import java.util.Arrays;
+
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
@@ -51,11 +52,11 @@ import com.google.common.base.Preconditions;
*
* @author yangli9
*/
-@SuppressWarnings({ "rawtypes", "unchecked" })
+@SuppressWarnings({"rawtypes", "unchecked"})
public class TrieDictionary<T> extends CacheDictionary<T> {
private static final long serialVersionUID = 1L;
- public static final byte[] MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
+ public static final byte[] MAGIC = new byte[]{0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74}; // "TrieDict"
public static final int MAGIC_SIZE_I = MAGIC.length;
public static final int BIT_IS_LAST_CHILD = 0x80;
@@ -78,6 +79,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
transient private long childOffsetMask;
transient private int firstByteOffset;
+
public TrieDictionary() { // default constructor for Writable interface
}
@@ -141,6 +143,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
return maxValueLength;
}
+
@Override
protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) {
int seq = lookupSeqNoFromValue(headSize, value, offset, offset + len, roundingFlag);
@@ -230,6 +233,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
return k;
}
+
@Override
protected byte[] getValueBytesFromIdWithoutCache(int id) {
byte[] buf = new byte[maxValueLength];
@@ -409,8 +413,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
new ObjectOutputStream(baos).writeObject(dict);
- TrieDictionary<String> dict2 = (TrieDictionary<String>) new ObjectInputStream(
- new ByteArrayInputStream(baos.toByteArray())).readObject();
+ TrieDictionary<String> dict2 = (TrieDictionary<String>) new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())).readObject();
Preconditions.checkArgument(dict.contains(dict2));
Preconditions.checkArgument(dict2.contains(dict));
Preconditions.checkArgument(dict.equals(dict2));
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 12ef117..1750ac1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -216,10 +216,7 @@ public class TrieDictionaryBuilder<T> {
// flatten trie footprint calculation, case of One-Byte-Per-Node
out.println("----------------------------------------------------------------------------");
- out.println("OBPN node size: "
- + (obpn_sizeValue + obpn_sizeNoValuesBeneath + obpn_sizeChildCount + obpn_sizeChildOffset) + " = "
- + obpn_sizeValue + " + " + obpn_sizeNoValuesBeneath + " + " + obpn_sizeChildCount + " + "
- + obpn_sizeChildOffset);
+ out.println("OBPN node size: " + (obpn_sizeValue + obpn_sizeNoValuesBeneath + obpn_sizeChildCount + obpn_sizeChildOffset) + " = " + obpn_sizeValue + " + " + obpn_sizeNoValuesBeneath + " + " + obpn_sizeChildCount + " + " + obpn_sizeChildOffset);
out.println("OBPN no. nodes: " + obpn_nNodes);
out.println("OBPN trie depth: " + maxValueLength);
out.println("OBPN footprint: " + obpn_footprint + " in bytes");
@@ -231,9 +228,7 @@ public class TrieDictionaryBuilder<T> {
out.println("MBPN total fan out: " + mbpn_nTotalFanOut);
out.println("MBPN average fan out: " + (double) mbpn_nTotalFanOut / mbpn_nChildLookups);
out.println("MBPN values size total: " + mbpn_sizeValueTotal);
- out.println("MBPN node size: "
- + (mbpn_sizeNoValueBytes + mbpn_sizeNoValueBeneath + mbpn_sizeChildOffset) + " = "
- + mbpn_sizeNoValueBytes + " + " + mbpn_sizeNoValueBeneath + " + " + mbpn_sizeChildOffset);
+ out.println("MBPN node size: " + (mbpn_sizeNoValueBytes + mbpn_sizeNoValueBeneath + mbpn_sizeChildOffset) + " = " + mbpn_sizeNoValueBytes + " + " + mbpn_sizeNoValueBeneath + " + " + mbpn_sizeChildOffset);
out.println("MBPN no. nodes: " + mbpn_nNodes);
out.println("MBPN trie depth: " + mbpn_trieDepth);
out.println("MBPN footprint: " + mbpn_footprint + " in bytes");
@@ -297,11 +292,9 @@ public class TrieDictionaryBuilder<T> {
s.obpn_sizeChildCount = 1;
s.obpn_sizeChildOffset = 5; // MSB used as isEndOfValue flag
s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total number of compressed bytes in OBPN
- s.obpn_footprint = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount
- + s.obpn_sizeChildOffset);
+ s.obpn_footprint = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
- long t = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount
- + s.obpn_sizeChildOffset - 1);
+ long t = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1);
if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2 because MSB of offset is used for isEndOfValue flag
s.obpn_sizeChildOffset--;
s.obpn_footprint = t;
@@ -314,11 +307,9 @@ public class TrieDictionaryBuilder<T> {
s.mbpn_sizeNoValueBytes = 1;
s.mbpn_sizeNoValueBeneath = BytesUtil.sizeForValue(s.nValues);
s.mbpn_sizeChildOffset = 5;
- s.mbpn_footprint = s.mbpn_sizeValueTotal
- + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
+ s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
- long t = s.mbpn_sizeValueTotal + s.mbpn_nNodes
- * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
+ long t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag
s.mbpn_sizeChildOffset--;
s.mbpn_footprint = t;
@@ -493,14 +484,12 @@ public class TrieDictionaryBuilder<T> {
}
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
- int flags = (int) trieBytes[parentOffset]
- & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
+ int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
trieBytes[parentOffset] |= flags;
}
- private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset,
- byte[] trieBytes) {
+ private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
int o = offset;
if (o > _2GB)
throw new IllegalStateException();
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 6ef767b..09d5bc2 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -63,12 +63,11 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
public TrieDictionaryForest(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, //
- ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) {
+ ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) {
init(trees, valueDivide, accuOffset, bytesConverter, baseId);
}
- private void init(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide,
- ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) {
+ private void init(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) {
this.trees = trees;
this.valueDivide = valueDivide;
this.accuOffset = accuOffset;
@@ -99,8 +98,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
@Override
- protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag)
- throws IllegalArgumentException {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
int index;
if (trees.size() == 1) {
index = 0;
@@ -111,8 +109,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
if (roundingFlag > 0) {
return getMinId(); //searching value smaller than the smallest value in dict
} else {
- throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' ("
- + Bytes.toStringBinary(value, offset, len) + ") not exists!");
+ throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
}
}
@@ -121,15 +118,13 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
if (search.compareTo(maxValueOfTree) > 0)
index++;
if (index >= trees.size())
- throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' ("
- + Bytes.toStringBinary(value, offset, len) + ") not exists!");
+ throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
}
}
TrieDictionary<T> tree = trees.get(index);
int id = tree.getIdFromValueBytesWithoutCache(value, offset, len, roundingFlag);
if (id == -1)
- throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' ("
- + Bytes.toStringBinary(value, offset, len) + ") not exists!");
+ throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
id = id + accuOffset.get(index);
id += baseId;
return id;
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
index 482c12a..0e5e63e 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
@@ -17,13 +17,13 @@
*/
package org.apache.kylin.dict;
-import java.util.ArrayList;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ByteArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+
/**
* Build a trie dictionary forest if the input values is ordered, or the forest falls back to a single trie.
*/
@@ -86,12 +86,10 @@ public class TrieDictionaryForestBuilder<T> {
return; //duplicate value
}
if (comp > 0) {
- logger.info("values not in ascending order, previous '{}', current '{}'", previousValue,
- valueByteArray);
+ logger.info("values not in ascending order, previous '{}', current '{}'", previousValue, valueByteArray);
isOrdered = false;
if (trees.size() > 0) {
- throw new IllegalStateException(
- "Invalid input data. Unordered data cannot be split into multi trees");
+ throw new IllegalStateException("Invalid input data. Unordered data cannot be split into multi trees");
}
}
}
@@ -112,8 +110,7 @@ public class TrieDictionaryForestBuilder<T> {
addTree(tree);
reset();
}
- TrieDictionaryForest<T> forest = new TrieDictionaryForest<T>(this.trees, this.valueDivide, this.accuOffset,
- this.bytesConverter, baseId);
+ TrieDictionaryForest<T> forest = new TrieDictionaryForest<T>(this.trees, this.valueDivide, this.accuOffset, this.bytesConverter, baseId);
// if input values are not in ascending order and tree num>1,TrieDictionaryForest can not work correctly.
if (forest.getTrees().size() > 1 && !isOrdered) {
throw new IllegalStateException("Invalid input data. Unordered data can not be split into multi trees");
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java
index 31e938a..ee3a2c2 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java
@@ -18,6 +18,11 @@
package org.apache.kylin.dict.global;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.dict.AppendTrieDictionary;
+import org.apache.kylin.dict.TrieDictionary;
+
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -27,11 +32,6 @@ import java.util.ArrayList;
import java.util.IdentityHashMap;
import java.util.LinkedList;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.AppendTrieDictionary;
-import org.apache.kylin.dict.TrieDictionary;
-
public class AppendDictNode {
public byte[] part;
public int id = -1;
@@ -195,14 +195,12 @@ public class AppendDictNode {
}
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
- int flags = (int) trieBytes[parentOffset]
- & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
+ int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
trieBytes[parentOffset] |= flags;
}
- private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId,
- byte[] trieBytes) {
+ private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId, byte[] trieBytes) {
int o = offset;
// childOffset
@@ -214,8 +212,7 @@ public class AppendDictNode {
// nValueBytes
if (n.part.length > 255)
- throw new RuntimeException(
- "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
+ throw new RuntimeException("Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
o++;
@@ -241,8 +238,7 @@ public class AppendDictNode {
@Override
public String toString() {
- return String.format("DictNode[root=%s, nodes=%d, firstValue=%s]", Bytes.toStringBinary(part), childrenCount,
- Bytes.toStringBinary(firstValue()));
+ return String.format("DictNode[root=%s, nodes=%d, firstValue=%s]", Bytes.toStringBinary(part), childrenCount, Bytes.toStringBinary(firstValue()));
}
static class Stats {
@@ -336,8 +332,7 @@ public class AppendDictNode {
s.mbpn_sizeValueTotal = s.nValueBytesCompressed + s.nValues * s.mbpn_sizeId;
s.mbpn_sizeNoValueBytes = 1;
s.mbpn_sizeChildOffset = 5;
- s.mbpn_footprint = s.mbpn_sizeValueTotal
- + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset);
+ s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset - 1);
// *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag
@@ -380,4 +375,4 @@ public class AppendDictNode {
}, 0);
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java
index a21832d..4e820e0 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java
@@ -18,6 +18,9 @@
package org.apache.kylin.dict.global;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
@@ -26,12 +29,8 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-
public class AppendDictSlice {
- static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44,
- 0x69, 0x63, 0x74 }; // "AppendTrieDict"
+ static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict"
static final int HEAD_SIZE_I = HEAD_MAGIC.length;
static final int BIT_IS_LAST_CHILD = 0x80;
static final int BIT_IS_END_OF_VALUE = 0x40;
@@ -59,8 +58,7 @@ public class AppendDictSlice {
throw new IllegalArgumentException("Wrong file type (magic does not match)");
try {
- DataInputStream headIn = new DataInputStream(
- new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
+ DataInputStream headIn = new DataInputStream(new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
this.headSize = headIn.readShort();
this.bodyLen = headIn.readInt();
this.nValues = headIn.readInt();
@@ -106,8 +104,7 @@ public class AppendDictSlice {
if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
break;
}
- nodeOffset = headSize
- + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
+ nodeOffset = headSize + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
if (nodeOffset == headSize) {
break;
}
@@ -158,8 +155,7 @@ public class AppendDictSlice {
} else if (comp < 0) { // try next child
if (checkFlag(c, BIT_IS_LAST_CHILD))
return -1;
- c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1)
- + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0);
+ c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1) + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0);
} else { // children are ordered by their first value byte
return -1;
}
@@ -265,8 +261,7 @@ public class AppendDictSlice {
@Override
public String toString() {
- return String.format("DictSlice[firstValue=%s, values=%d, bytes=%d]", Bytes.toStringBinary(getFirstValue()),
- nValues, bodyLen);
+ return String.format("DictSlice[firstValue=%s, values=%d, bytes=%d]", Bytes.toStringBinary(getFirstValue()), nValues, bodyLen);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java
index d61258a..323fe6b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java
@@ -18,13 +18,13 @@
package org.apache.kylin.dict.global;
+import org.apache.kylin.common.util.Bytes;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.kylin.common.util.Bytes;
-
public class AppendDictSliceKey implements Comparable<AppendDictSliceKey> {
static final AppendDictSliceKey START_KEY = AppendDictSliceKey.wrap(new byte[0]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
index 2911798..90d65b6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
@@ -18,18 +18,18 @@
package org.apache.kylin.dict.global;
-import static com.google.common.base.Preconditions.checkState;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.TreeMap;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.AppendTrieDictionary;
import org.apache.kylin.dict.BytesConverter;
import org.apache.kylin.dict.StringBytesConverter;
+import java.io.IOException;
+import java.util.List;
+import java.util.TreeMap;
+
+import static com.google.common.base.Preconditions.checkState;
+
public class AppendTrieDictionaryBuilder {
private final String baseDir;
@@ -47,8 +47,7 @@ public class AppendTrieDictionaryBuilder {
private AppendDictNode curNode;
public AppendTrieDictionaryBuilder(String resourceDir, int maxEntriesPerSlice) throws IOException {
- this.baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourceDir
- + "/";
+ this.baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourceDir + "/";
this.workingDir = this.baseDir + "/working";
this.maxEntriesPerSlice = maxEntriesPerSlice;
init();
@@ -84,8 +83,7 @@ public class AppendTrieDictionaryBuilder {
curNode = new AppendDictNode(new byte[0], false);
sliceFileMap.put(AppendDictSliceKey.START_KEY, null);
}
- checkState(sliceFileMap.firstKey().equals(AppendDictSliceKey.START_KEY),
- "first key should be \"\", but got \"%s\"", sliceFileMap.firstKey());
+ checkState(sliceFileMap.firstKey().equals(AppendDictSliceKey.START_KEY), "first key should be \"\", but got \"%s\"", sliceFileMap.firstKey());
AppendDictSliceKey nextKey = sliceFileMap.floorKey(AppendDictSliceKey.wrap(valueBytes));
@@ -123,8 +121,7 @@ public class AppendTrieDictionaryBuilder {
flushCurrentNode();
}
- GlobalDictMetadata metadata = new GlobalDictMetadata(baseId, this.maxId, this.maxValueLength, this.nValues,
- this.bytesConverter, sliceFileMap);
+ GlobalDictMetadata metadata = new GlobalDictMetadata(baseId, this.maxId, this.maxValueLength, this.nValues, this.bytesConverter, sliceFileMap);
store.commit(workingDir, metadata);
AppendTrieDictionary dict = new AppendTrieDictionary();
@@ -161,8 +158,7 @@ public class AppendTrieDictionaryBuilder {
}
} else {
// otherwise, split the current node into two
- AppendDictNode c = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue,
- node.children);
+ AppendDictNode c = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
c.id = node.id;
node.reset(BytesUtil.subarray(node.part, 0, i), true);
node.addChild(c);
@@ -174,8 +170,7 @@ public class AppendTrieDictionaryBuilder {
// if partially matched the current, split the current node, add the new
// value, make a 3-way
if (i < n) {
- AppendDictNode c1 = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue,
- node.children);
+ AppendDictNode c1 = new AppendDictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
c1.id = node.id;
AppendDictNode c2 = addNodeMaybeOverflow(value, j, nn);
node.reset(BytesUtil.subarray(node.part, 0, i), false);