You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/06 13:03:03 UTC
incubator-kylin git commit: KYLIN-1116 use local dictionary for
inverted index building
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1116 82bfa924c -> 525294387
KYLIN-1116 use local dictionary for inverted index building
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/52529438
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/52529438
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/52529438
Branch: refs/heads/KYLIN-1116
Commit: 525294387f0f6fec6489d5e6a202105b05e8459c
Parents: 82bfa92
Author: shaofengshi <sh...@apache.org>
Authored: Fri Nov 6 20:02:30 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Nov 6 20:02:30 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/BuildIIWithStreamTest.java | 2 +-
.../org/apache/kylin/job/hadoop/invertedindex/IITest.java | 2 +-
.../engine/mr/invertedindex/InvertedIndexReducer.java | 2 +-
.../apache/kylin/invertedindex/index/SliceBuilder.java | 6 ++----
.../java/org/apache/kylin/invertedindex/model/IIDesc.java | 10 ----------
5 files changed, 5 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52529438/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 2d40b09..ddfd399 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -190,7 +190,7 @@ public class BuildIIWithStreamTest {
ToolRunner.run(new IICreateHTableJob(), args);
final IIDesc iiDesc = segment.getIIDesc();
- final SliceBuilder sliceBuilder = new SliceBuilder(desc, (short) 0, iiDesc.isUseLocalDictionary());
+ final SliceBuilder sliceBuilder = new SliceBuilder(desc, (short) 0);
List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
int count = sorted.size();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52529438/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index a393ce3..ac91efb 100644
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -102,7 +102,7 @@ public class IITest extends LocalFileMetadataTestCase {
StreamingBatch batch = new StreamingBatch(streamingMessages, Pair.newPair(0L, System.currentTimeMillis()));
iiRows = Lists.newArrayList();
- final Slice slice = new SliceBuilder(iiDesc, (short) 0, true).buildSlice((batch));
+ final Slice slice = new SliceBuilder(iiDesc, (short) 0).buildSlice((batch));
IIKeyValueCodec codec = new IIKeyValueCodec(slice.getInfo());
for (IIRow iiRow : codec.encodeKeyValue(slice)) {
iiRows.add(iiRow);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52529438/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
index 56c0b9e..c223159 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
@@ -73,7 +73,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, Object, Imm
sliceSize = ii.getDescriptor().getSliceSize();
short shard = (short) context.getTaskAttemptID().getTaskID().getId();
System.out.println("Generating to shard - " + shard);
- sliceBuilder = new SliceBuilder(seg.getIIDesc(), shard, true);
+ sliceBuilder = new SliceBuilder(seg.getIIDesc(), shard);
messages = Lists.newArrayListWithCapacity(sliceSize);
immutableBytesWritable = new ImmutableBytesWritable();
valueBuf = ByteBuffer.allocate(1024 * 1024); // 1MB
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52529438/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index 1c293d7..ef63b0f 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -38,12 +38,10 @@ public final class SliceBuilder {
private final BatchSliceMaker sliceMaker;
private final IIDesc iiDesc;
- private final boolean useLocalDict;
- public SliceBuilder(IIDesc desc, short shard, boolean useLocalDict) {
+ public SliceBuilder(IIDesc desc, short shard) {
this.iiDesc = desc;
this.sliceMaker = new BatchSliceMaker(desc, shard);
- this.useLocalDict = useLocalDict;
}
public Slice buildSlice(StreamingBatch microStreamBatch) {
@@ -54,7 +52,7 @@ public final class SliceBuilder {
return input.getData();
}
});
- final Dictionary<?>[] dictionaries = useLocalDict ? IIDictionaryBuilder.buildDictionary(messages, iiDesc) : new Dictionary[iiDesc.listAllColumns().size()];
+ final Dictionary<?>[] dictionaries = IIDictionaryBuilder.buildDictionary(messages, iiDesc);
TableRecordInfo tableRecordInfo = new TableRecordInfo(iiDesc, dictionaries);
return build(messages, tableRecordInfo, dictionaries);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52529438/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index bfa4eaa..71737dc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -78,8 +78,6 @@ public class IIDesc extends RootPersistentEntity {
private short sharding = 1; // parallelism
@JsonProperty("slice_size")
private int sliceSize = 50000; // no. rows
- @JsonProperty("useLocalDictionary")
- private boolean useLocalDictionary = true;
@JsonProperty("engine_type")
private int engineType = IEngineAware.ID_MR_II;
@@ -374,14 +372,6 @@ public class IIDesc extends RootPersistentEntity {
this.name = name;
}
- public boolean isUseLocalDictionary() {
- return useLocalDictionary;
- }
-
- public void setUseLocalDictionary(boolean useLocalDictionary) {
- this.useLocalDictionary = useLocalDictionary;
- }
-
public String calculateSignature() {
MessageDigest md = null;
try {