You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/31 10:22:08 UTC
[2/2] kylin git commit: KYLIN-1851 Link into DictionaryGenerator,
half way
KYLIN-1851 Link into DictionaryGenerator, half way
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3fedee9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3fedee9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3fedee9f
Branch: refs/heads/KYLIN-1851
Commit: 3fedee9fe76c979572594b85fbc6a660eb61f963
Parents: 0359fb3
Author: Li Yang <li...@apache.org>
Authored: Mon Oct 31 18:21:36 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Oct 31 18:21:43 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/DictionaryGenerator.java | 4 ++--
.../apache/kylin/dict/TrieDictionaryForestBuilder.java | 4 ++++
.../java/org/apache/kylin/dict/lookup/SnapshotTable.java | 11 ++++++-----
3 files changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fedee9f/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 0adf40e..f2c4507 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -137,7 +137,7 @@ public class DictionaryGenerator {
private static class StringDictBuilder implements IDictionaryBuilder {
@Override
public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter());
+ TrieDictionaryForestBuilder builder = new TrieDictionaryForestBuilder(new StringBytesConverter(), baseId);
byte[] value;
while (valueEnumerator.moveNext()) {
value = valueEnumerator.current();
@@ -148,7 +148,7 @@ public class DictionaryGenerator {
if (returnSamples.size() < nSamples && returnSamples.contains(v) == false)
returnSamples.add(v);
}
- return builder.build(baseId);
+ return builder.build();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fedee9f/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
index 494ee71..4cc74ce 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
@@ -49,6 +49,10 @@ public class TrieDictionaryForestBuilder<T> {
private int curOffset;
+ public TrieDictionaryForestBuilder(BytesConverter<T> bytesConverter) {
+ this(bytesConverter, 0);
+ }
+
public TrieDictionaryForestBuilder(BytesConverter<T> bytesConverter, int baseId) {
this.bytesConverter = bytesConverter;
this.trieBuilder = new TrieDictionaryBuilder<T>(bytesConverter);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fedee9f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index db1a170..a60b70d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import com.google.common.base.Strings;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -36,6 +35,7 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.dict.TrieDictionaryForestBuilder;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.ReadableTable;
@@ -43,6 +43,7 @@ import org.apache.kylin.source.ReadableTable;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Strings;
/**
* @author yangli9
@@ -76,7 +77,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
int maxIndex = tableDesc.getMaxColumnIndex();
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter());
TableReader reader = table.getReader();
try {
@@ -95,7 +96,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
IOUtils.closeQuietly(reader);
}
- this.dict = b.build(0);
+ this.dict = b.build();
ArrayList<int[]> allRowIndices = new ArrayList<int[]>();
reader = table.getReader();
@@ -253,7 +254,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
}
} else {
List<String[]> rows = new ArrayList<String[]>(rowNum);
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter());
for (int i = 0; i < rowNum; i++) {
String[] row = new String[n];
@@ -267,7 +268,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
b.addValue(row[j]);
}
}
- this.dict = b.build(0);
+ this.dict = b.build();
for (String[] row : rows) {
int[] rowIndex = new int[n];
for (int i = 0; i < n; i++) {