You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/02/16 07:11:53 UTC
[1/6] kylin git commit: KYLIN-1420 Query returns empty result on
partition column's boundary condition
Repository: kylin
Updated Branches:
refs/heads/helix-201602 3b4c9928a -> 01cda3cbd
KYLIN-1420 Query returns empty result on partition column's boundary condition
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc460de2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc460de2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc460de2
Branch: refs/heads/helix-201602
Commit: dc460de21327da50154ac7b214ae877dde4d8055
Parents: 3b4c992
Author: shaofengshi <sh...@apache.org>
Authored: Mon Feb 15 15:52:15 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Feb 16 11:02:42 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dc460de2/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index a72426d..d314dde 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -108,8 +108,8 @@ public class GTScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (partitionColRef != null && range.column.equals(partitionColRef)) {
if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), range.end) <= 0 //
- && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) < 0) {
- //segment range is [Closed,Open)
+ && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) <= 0) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <=.
} else {
logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}",//
new Object[] { partitionColRef, makeReadable(segmentStartAndEnd.getFirst()), makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) });
[2/6] kylin git commit: KYLIN-1387 should support empty segment
Posted by sh...@apache.org.
KYLIN-1387 should support empty segment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95f8a2c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95f8a2c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95f8a2c5
Branch: refs/heads/helix-201602
Commit: 95f8a2c567875ed48fc6de27a3918e8d6d607835
Parents: dc460de
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 09:42:34 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Feb 16 11:05:12 2016 +0800
----------------------------------------------------------------------
.../hbase/steps/SequenceFileCuboidWriter.java | 48 +++++++++++---------
1 file changed, 26 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/95f8a2c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 4d76522..8c2d5e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -25,35 +25,39 @@ public class SequenceFileCuboidWriter extends KVGTRecordWriter {
public SequenceFileCuboidWriter(CubeDesc cubeDesc, CubeSegment segment) {
super(cubeDesc, segment);
+ try {
+ initiate();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
-
- @Override
- protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
+ protected void initiate() throws IOException {
if (writer == null) {
- synchronized (SequenceFileCuboidWriter.class) {
- if (writer == null) {
- JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
- String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
- Path cuboidPath = new Path(cuboidRoot);
- FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
- try {
- if (fs.exists(cuboidPath)) {
- fs.delete(cuboidPath, true);
- }
-
- fs.mkdirs(cuboidPath);
- } finally {
- IOUtils.closeQuietly(fs);
- }
-
- Path cuboidFile = new Path(cuboidPath, "data.seq");
- logger.debug("Cuboid is written to " + cuboidFile);
- writer = SequenceFile.createWriter(HadoopUtil.getCurrentConfiguration(), SequenceFile.Writer.file(cuboidFile), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class));
+ JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
+ String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
+ Path cuboidPath = new Path(cuboidRoot);
+ FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
+ try {
+ if (fs.exists(cuboidPath)) {
+ fs.delete(cuboidPath, true);
}
+
+ fs.mkdirs(cuboidPath);
+ } finally {
+ IOUtils.closeQuietly(fs);
}
+
+ Path cuboidFile = new Path(cuboidPath, "data.seq");
+ logger.debug("Cuboid is written to " + cuboidFile);
+ writer = SequenceFile.createWriter(HadoopUtil.getCurrentConfiguration(), SequenceFile.Writer.file(cuboidFile), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class));
}
+ }
+
+ @Override
+ protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
+
Text outputValue = new Text();
Text outputKey = new Text();
outputKey.set(key.array(), key.offset(), key.length());
[4/6] kylin git commit: KYLIN-1420 enhance and update test case
Posted by sh...@apache.org.
KYLIN-1420 enhance and update test case
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0d630987
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0d630987
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0d630987
Branch: refs/heads/helix-201602
Commit: 0d630987694d6af4afec3f02364f8d5540d15b82
Parents: 9bd272c
Author: shaofengshi <sh...@apache.org>
Authored: Mon Feb 15 18:06:18 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Feb 16 14:05:54 2016 +0800
----------------------------------------------------------------------
.../kylin/gridtable/GTScanRangePlanner.java | 27 ++++++++------------
.../kylin/gridtable/DictGridTableTest.java | 2 +-
2 files changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0d630987/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index d314dde..559a245 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -1,17 +1,8 @@
package org.apache.kylin.gridtable;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -26,9 +17,7 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import java.util.*;
public class GTScanRangePlanner {
@@ -108,8 +97,10 @@ public class GTScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (partitionColRef != null && range.column.equals(partitionColRef)) {
if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), range.end) <= 0 //
- && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) <= 0) {
- //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <=.
+ && (rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) < 0 //
+ || rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) == 0 //
+ && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in condition.
} else {
logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}",//
new Object[] { partitionColRef, makeReadable(segmentStartAndEnd.getFirst()), makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) });
@@ -346,9 +337,11 @@ public class GTScanRangePlanner {
private ByteArray begin = ByteArray.EMPTY;
private ByteArray end = ByteArray.EMPTY;
private Set<ByteArray> valueSet;
+ private FilterOperatorEnum op;
public ColumnRange(TblColRef column, Set<ByteArray> values, FilterOperatorEnum op) {
this.column = column;
+ this.op = op;
switch (op) {
case EQ:
http://git-wip-us.apache.org/repos/asf/kylin/blob/0d630987/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index df69c17..674aa15 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -118,7 +118,7 @@ public class DictGridTableTest {
{
LogicalTupleFilter filter = and(timeComp4, ageComp1);
List<GTScanRange> r = planner.planScanRanges(filter);
- assertEquals(0, r.size());
+ assertEquals(1, r.size());
}
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
[5/6] kylin git commit: KYLIN-1417 Change to use TreeMap to allow
null as value
Posted by sh...@apache.org.
KYLIN-1417 Change to use TreeMap to allow null as value
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6fcfda8a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6fcfda8a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6fcfda8a
Branch: refs/heads/helix-201602
Commit: 6fcfda8a09529008db23f5aee6c10b5c7a7962ad
Parents: 0d63098
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 10:29:47 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Feb 16 14:06:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/kafka/TimedJsonStreamParser.java | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6fcfda8a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index b075387..4dccb17 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -35,11 +35,7 @@
package org.apache.kylin.source.kafka;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -106,7 +102,7 @@ public final class TimedJsonStreamParser extends StreamingParser {
public StreamingMessage parse(MessageAndOffset messageAndOffset) {
try {
Map<String, String> message = mapper.readValue(new ByteBufferBackedInputStream(messageAndOffset.message().payload()), mapType);
- ConcurrentMap<String, String> root = new ConcurrentSkipListMap<String, String>(String.CASE_INSENSITIVE_ORDER);
+ Map<String, String> root = new TreeMap<String, String>(String.CASE_INSENSITIVE_ORDER);
root.putAll(message);
String tsStr = root.get(tsColName);
//Preconditions.checkArgument(!StringUtils.isEmpty(tsStr), "Timestamp field " + tsColName + //
[6/6] kylin git commit: KYLIN-1421 fix the “Last build time” is always empty issue
Posted by sh...@apache.org.
KYLIN-1421 fix the “Last build time” is always empty issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01cda3cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01cda3cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01cda3cb
Branch: refs/heads/helix-201602
Commit: 01cda3cbd1308713a351812a4515d4f364dda7f9
Parents: 6fcfda8
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 14:07:00 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Feb 16 14:09:38 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/01cda3cb/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index d373e4a..4b11408 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -117,6 +117,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
CubeSegment segment = cubeManager.appendSegments(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond(), false, false);
segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
segment.setInputRecords(streamingBatch.getMessages().size());
+ segment.setLastBuildTime(System.currentTimeMillis());
return segment;
} catch (IOException e) {
throw new RuntimeException("failed to create IBuildable", e);
[3/6] kylin git commit: KYLIN-1421 Cube "source record" is always
zero for streaming
Posted by sh...@apache.org.
KYLIN-1421 Cube "source record" is always zero for streaming
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd272c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd272c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd272c8
Branch: refs/heads/helix-201602
Commit: 9bd272c813e4f350c637261efc7f8df7658a4882
Parents: 95f8a2c
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 10:30:05 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Feb 16 11:05:20 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd272c8/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index 20ff01d..d373e4a 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -116,6 +116,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
try {
CubeSegment segment = cubeManager.appendSegments(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond(), false, false);
segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
+ segment.setInputRecords(streamingBatch.getMessages().size());
return segment;
} catch (IOException e) {
throw new RuntimeException("failed to create IBuildable", e);