You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:00:36 UTC
[01/50] [abbrv] incubator-kylin git commit: Merge pull request #446
from janzhongi/staging
Repository: incubator-kylin
Updated Branches:
refs/heads/streaming-localdict [created] af153c42f
Merge pull request #446 from janzhongi/staging
fix filter 0 bytes bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8cca80d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8cca80d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8cca80d5
Branch: refs/heads/streaming-localdict
Commit: 8cca80d570c64c71050fad206492b623e43c9d77
Parents: 65445e8 7dae30e
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 15:27:18 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 15:27:18 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/filters/filter.js | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
[10/50] [abbrv] incubator-kylin git commit: Merge branch 'staging'
into streaming
Posted by li...@apache.org.
Merge branch 'staging' into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e3c50724
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e3c50724
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e3c50724
Branch: refs/heads/streaming-localdict
Commit: e3c50724379202e725a7d7c9d25f293009400ebf
Parents: bb83ca1 5f9e2c9
Author: jiazhong <ji...@ebay.com>
Authored: Tue Mar 10 18:20:16 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Mar 10 18:20:16 2015 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
14 files changed, 115 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
[31/50] [abbrv] incubator-kylin git commit: Merge remote-tracking
branch 'origin/master' into streaming-localdict
Posted by li...@apache.org.
Merge remote-tracking branch 'origin/master' into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7b48b51f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7b48b51f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7b48b51f
Branch: refs/heads/streaming-localdict
Commit: 7b48b51f68994d228d8d701f9cc14e97b02eb53d
Parents: a1b0177 2efaf25
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 16:01:36 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 16:01:36 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[02/50] [abbrv] incubator-kylin git commit: stream build
Posted by li...@apache.org.
stream build
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3cc984f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3cc984f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3cc984f6
Branch: refs/heads/streaming-localdict
Commit: 3cc984f651aa2eeab12a9039e72569d999ddff4d
Parents: cb6d24b
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Mar 10 16:56:20 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Mar 10 16:56:20 2015 +0800
----------------------------------------------------------------------
.../test_kylin_ii_inner_join_desc.json | 4 +-
.../test_kylin_ii_left_join_desc.json | 4 +-
.../invertedindex/index/TableRecordInfo.java | 1 +
.../kylin/invertedindex/model/IIDesc.java | 2 +
job/pom.xml | 6 +++
.../apache/kylin/streaming/StreamBuilder.java | 41 ++++++++++++-------
.../invertedindex/IIStreamBuilder.java | 43 ++++++++++++--------
7 files changed, 64 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
index eb896a0..18958b6 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
@@ -65,6 +65,6 @@
"PRICE",
"ITEM_COUNT"
],
- "sharding": 4,
- "slice_size": 50000
+ "sharding": 1,
+ "slice_size": 1000
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
index 424b688..654cd38 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
@@ -65,6 +65,6 @@
"PRICE",
"ITEM_COUNT"
],
- "sharding": 4,
- "slice_size": 50000
+ "sharding": 1,
+ "slice_size": 1000
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index cc229da..68feb03 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -107,6 +107,7 @@ public class TableRecordInfo {
for (int i = 0; i < nColumns; ++i) {
final TblColRef tblColRef = getColumns().get(i);
isMetric[i] = desc.isMetricsCol(i);
+ dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
lengths[i] = measureCodecMap.get(tblColRef).getLength();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index e094dfc..142ac39 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -53,8 +53,10 @@ public class IIDesc extends RootPersistentEntity {
public static final String HBASE_FAMILY = "f";
public static final String HBASE_QUALIFIER = "c";
+ public static final String HBASE_DICTIONARY = "d";
public static final byte[] HBASE_FAMILY_BYTES = Bytes.toBytes(HBASE_FAMILY);
public static final byte[] HBASE_QUALIFIER_BYTES = Bytes.toBytes(HBASE_QUALIFIER);
+ public static final byte[] HBASE_DICTIONARY_BYTES = Bytes.toBytes(HBASE_DICTIONARY);
private KylinConfig config;
private DataModelDesc model;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/job/pom.xml
----------------------------------------------------------------------
diff --git a/job/pom.xml b/job/pom.xml
index 1fff5af..45c47b3 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -58,6 +58,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-streaming</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
index 005c255..d86da90 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
/**
* Created by qianzhou on 2/17/15.
@@ -48,27 +49,21 @@ public abstract class StreamBuilder implements Runnable {
private static final Logger logger = LoggerFactory.getLogger(StreamBuilder.class);
- private static final int BATCH_BUILD_BYTES_THRESHOLD = 64 * 1024;
private static final int BATCH_BUILD_INTERVAL_THRESHOLD = 5 * 60 * 1000;
+ private final int sliceSize;
private BlockingQueue<Stream> streamQueue;
private long lastBuildTime = System.currentTimeMillis();
- private int bytesTotal = 0;
- public StreamBuilder(BlockingQueue<Stream> streamQueue) {
+ public StreamBuilder(BlockingQueue<Stream> streamQueue, int sliceSize) {
this.streamQueue = streamQueue;
+ this.sliceSize = sliceSize;
}
- protected abstract void build(List<Stream> streamsToBuild);
-
- private void buildStream(List<Stream> streams) {
- build(streams);
- clearCounter();
- }
+ protected abstract boolean build(List<Stream> streamsToBuild);
private void clearCounter() {
lastBuildTime = System.currentTimeMillis();
- bytesTotal = 0;
}
@Override
@@ -77,13 +72,29 @@ public abstract class StreamBuilder implements Runnable {
List<Stream> streamToBuild = Lists.newArrayList();
clearCounter();
while (true) {
- final Stream stream = streamQueue.take();
+ final Stream stream = streamQueue.poll(200, TimeUnit.MILLISECONDS);
+ if (stream == null) {
+ continue;
+ } else {
+ if (stream.getOffset() < 0) {
+ if (!streamToBuild.isEmpty()) {
+ build(streamToBuild);
+ }
+ logger.warn("streaming encountered EOF, stop building");
+ break;
+ }
+ }
streamToBuild.add(stream);
- bytesTotal += stream.getRawData().length;
- if (bytesTotal >= BATCH_BUILD_BYTES_THRESHOLD) {
- buildStream(streamToBuild);
+ if (streamToBuild.size() >= this.sliceSize) {
+ if (build(streamToBuild)) {
+ clearCounter();
+ streamToBuild.clear();
+ }
} else if ((System.currentTimeMillis() - lastBuildTime) > BATCH_BUILD_INTERVAL_THRESHOLD) {
- buildStream(streamToBuild);
+ if (build(streamToBuild)) {
+ clearCounter();
+ streamToBuild.clear();
+ }
} else {
continue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc984f6/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index b2683b4..8b2673d 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -36,15 +36,14 @@ package org.apache.kylin.streaming.invertedindex;
import com.google.common.base.Function;
import com.google.common.collect.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.lookup.ReadableTable;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.SliceBuilder;
import org.apache.kylin.invertedindex.index.TableRecord;
@@ -54,7 +53,6 @@ import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.streaming.KafkaConfig;
import org.apache.kylin.streaming.Stream;
import org.apache.kylin.streaming.StreamBuilder;
import org.slf4j.Logger;
@@ -65,7 +63,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
/**
* Created by qianzhou on 3/3/15.
@@ -75,15 +73,23 @@ public class IIStreamBuilder extends StreamBuilder {
private static Logger logger = LoggerFactory.getLogger(IIStreamBuilder.class);
private IIDesc desc = null;
- private KafkaConfig kafkaConfig = null;
private HTableInterface hTable = null;
+ private int partitionId = -1;
- public IIStreamBuilder(BlockingQueue<Stream> streamQueue) {
- super(streamQueue);
+ public IIStreamBuilder(LinkedBlockingDeque<Stream> queue, String hTableName, IIDesc desc, int partitionId) {
+ super(queue, desc.getSliceSize());
+ this.desc = desc;
+ this.partitionId = partitionId;
+ try {
+ this.hTable = HConnectionManager.createConnection(HBaseConfiguration.create()).getTable(hTableName);
+ } catch (IOException e) {
+ logger.error("cannot open htable name:" + hTableName, e);
+ throw new RuntimeException("cannot open htable name:" + hTableName, e);
+ }
}
@Override
- protected void build(List<Stream> streamsToBuild) {
+ protected boolean build(List<Stream> streamsToBuild) {
List<List<String>> table = Lists.transform(streamsToBuild, new Function<Stream, List<String>>() {
@Nullable
@Override
@@ -101,14 +107,16 @@ public class IIStreamBuilder extends StreamBuilder {
}
}
TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap, measureCodecMap);
- SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) kafkaConfig.getPartitionId());
+ SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
- try {
- loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()));
- submitOffset();
- } catch (IOException e) {
- logger.error("error load to hbase, build failed", e);
- }
+// try {
+// loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()));
+// submitOffset();
+// } catch (IOException e) {
+// logger.error("error load to hbase, build failed", e);
+// return false;
+// }
+ return true;
}
private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
@@ -139,8 +147,7 @@ public class IIStreamBuilder extends StreamBuilder {
}
private List<String> parseStream(Stream stream, IIDesc desc) {
- List<String> result = Lists.newArrayList();
- return result;
+ return Lists.newArrayList(new String(stream.getRawData()).split(","));
}
private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo) {
[09/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1142b686
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1142b686
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1142b686
Branch: refs/heads/streaming-localdict
Commit: 1142b6865e948d7a72feb0344935d82c77a0535d
Parents: 132b0ca
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Mar 10 17:33:02 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Mar 10 17:33:02 2015 +0800
----------------------------------------------------------------------
.../invertedindex/model/IIKeyValueCodec.java | 31 ++++------
.../kylin/invertedindex/model/KeyValuePair.java | 65 ++++++++++++++++++++
.../invertedindex/InvertedIndexReducer.java | 10 +--
.../apache/kylin/streaming/StreamBuilder.java | 2 +-
.../invertedindex/IIStreamBuilder.java | 4 +-
5 files changed, 87 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1142b686/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 8bb6e88..070b672 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -46,48 +46,43 @@ public class IIKeyValueCodec {
this.infoDigest = digest;
}
- public Collection<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKeyValue(
- Slice slice) {
- ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result = Lists
+ public Collection<KeyValuePair> encodeKeyValue(Slice slice) {
+ ArrayList<KeyValuePair> result = Lists
.newArrayList();
ColumnValueContainer[] containers = slice.getColumnValueContainers();
for (int col = 0; col < containers.length; col++) {
if (containers[col] instanceof BitMapContainer) {
- collectKeyValues(slice, col, (BitMapContainer) containers[col],
- result);
+ result.addAll(collectKeyValues(slice, col, (BitMapContainer) containers[col]));
} else if (containers[col] instanceof CompressedValueContainer) {
- collectKeyValues(slice, col,
- (CompressedValueContainer) containers[col], result);
+ result.add(collectKeyValues(slice, col, (CompressedValueContainer) containers[col]));
} else {
- throw new IllegalArgumentException("Unkown container class "
+ throw new IllegalArgumentException("Unknown container class "
+ containers[col].getClass());
}
}
return result;
}
- private void collectKeyValues(Slice slice,
+ private KeyValuePair collectKeyValues(Slice slice,
int col,
- CompressedValueContainer container, //
- ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result) {
+ CompressedValueContainer container) {
ImmutableBytesWritable key = encodeKey(slice.getShard(),
slice.getTimestamp(), col, -1);
ImmutableBytesWritable value = container.toBytes();
- result.add(new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(
- key, value));
+ return new KeyValuePair(col, key, value);
}
- private void collectKeyValues(Slice slice,
+ private List<KeyValuePair> collectKeyValues(Slice slice,
int col,
- BitMapContainer container, //
- ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result) {
+ BitMapContainer container) {
List<ImmutableBytesWritable> values = container.toBytes();
+ ArrayList<KeyValuePair> list = Lists.newArrayListWithExpectedSize(values.size());
for (int v = 0; v < values.size(); v++) {
ImmutableBytesWritable key = encodeKey(slice.getShard(),
slice.getTimestamp(), col, v);
- result.add(new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(
- key, values.get(v)));
+ list.add(new KeyValuePair(col, key, values.get(v)));
}
+ return list;
}
ImmutableBytesWritable encodeKey(short shard, long timestamp, int col,
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1142b686/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
new file mode 100644
index 0000000..445389f
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+
+/**
+ * Created by qianzhou on 3/10/15.
+ */
+public final class KeyValuePair {
+
+ private int columnIndex;
+ private ImmutableBytesWritable key;
+ private ImmutableBytesWritable value;
+
+ public KeyValuePair(int columnIndex, ImmutableBytesWritable key, ImmutableBytesWritable value) {
+ this.columnIndex = columnIndex;
+ this.key = key;
+ this.value = value;
+ }
+
+ public int getColumnIndex() {
+ return columnIndex;
+ }
+
+ public ImmutableBytesWritable getKey() {
+ return key;
+ }
+
+ public ImmutableBytesWritable getValue() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1142b686/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
index f65db0e..40cca36 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
@@ -18,11 +18,8 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.mr.KylinReducer;
@@ -34,10 +31,13 @@ import org.apache.kylin.invertedindex.index.SliceBuilder;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.invertedindex.model.KeyValuePair;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import java.io.IOException;
+
/**
* @author yangli9
*/
@@ -92,8 +92,8 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
}
private void output(Slice slice, Context context) throws IOException, InterruptedException {
- for (Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair : kv.encodeKeyValue(slice)) {
- context.write(pair.getFirst(), pair.getSecond());
+ for (KeyValuePair pair : kv.encodeKeyValue(slice)) {
+ context.write(pair.getKey(), pair.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1142b686/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
index d86da90..a86a9b1 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
@@ -72,7 +72,7 @@ public abstract class StreamBuilder implements Runnable {
List<Stream> streamToBuild = Lists.newArrayList();
clearCounter();
while (true) {
- final Stream stream = streamQueue.poll(200, TimeUnit.MILLISECONDS);
+ final Stream stream = streamQueue.poll(50, TimeUnit.MILLISECONDS);
if (stream == null) {
continue;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1142b686/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 8b2673d..6339fe4 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -168,7 +168,9 @@ public class IIStreamBuilder extends StreamBuilder {
final byte[] key = pair.getFirst().get();
final byte[] value = pair.getSecond().get();
Put put = new Put(key);
- put.add("cf".getBytes(), "qn".getBytes(), value);
+ put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
+ //dictionary
+// put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
data.add(put);
}
hTable.put(data);
[17/50] [abbrv] incubator-kylin git commit: KYLIN-629 Kylin failed to
run mapreduce job if there is no mapreduce.application.classpath in
mapred-site.xml
Posted by li...@apache.org.
KYLIN-629 Kylin failed to run mapreduce job if there is no mapreduce.application.classpath in mapred-site.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a5821c83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a5821c83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a5821c83
Branch: refs/heads/streaming-localdict
Commit: a5821c83f8393853dba271522c2c96bd912f0b0e
Parents: 93f1f26
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 11:21:26 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 11:21:26 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/AbstractHadoopJob.java | 64 ++++++++++++++------
1 file changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5821c83/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index ff19357..087dd04 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -23,18 +23,6 @@ package org.apache.kylin.job.hadoop;
*
*/
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.StringSplitter;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.job.tools.OptionsHelper;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -50,9 +38,22 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.cmd.ShellCmdOutput;
+import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.job.tools.OptionsHelper;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +64,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.hadoop.util.StringUtils.formatTime;
+
@SuppressWarnings("static-access")
public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Logger logger = LoggerFactory.getLogger(AbstractHadoopJob.class);
@@ -121,7 +124,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
} else {
job.waitForCompletion(true);
retVal = job.isSuccessful() ? 0 : 1;
- logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + StringUtils.formatTime((System.nanoTime() - start) / 1000000L));
+ logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L));
}
return retVal;
}
@@ -135,7 +138,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
System.exit(5);
}
}
-
+
private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
protected void setJobClasspath(Job job) {
@@ -154,16 +157,37 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// yarn classpath is comma separated
kylinHiveDependency = kylinHiveDependency.replace(":", ",");
Configuration jobConf = job.getConfiguration();
- final String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
- if (classpath == null) {
- jobConf.set(MAP_REDUCE_CLASSPATH, kylinHiveDependency);
- } else {
- jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
+ String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+ if (classpath == null || classpath.length() == 0) {
+ logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
+ classpath = getDefaultMapRedClasspath();
+ logger.info("The default mapred classpath is: " + classpath);
}
+
+ jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
+
}
logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
}
+
+ private String getDefaultMapRedClasspath() {
+
+ String classpath = "";
+ try {
+ CliCommandExecutor executor = KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
+ ShellCmdOutput output = new ShellCmdOutput();
+ executor.execute("mapred classpath", output);
+
+ classpath = output.getOutput();
+ } catch (IOException e) {
+ logger.error("Failed to run: 'mapred classpath'.", e);
+ }
+
+ return classpath;
+ }
+
+
public void addInputDirs(String input, Job job) throws IOException {
for (String inp : StringSplitter.split(input, ",")) {
inp = inp.trim();
[23/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e07c7020
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e07c7020
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e07c7020
Branch: refs/heads/streaming-localdict
Commit: e07c702061df5656735578a2f400ce3770e384cf
Parents: 63695e4
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 16:50:40 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 16:50:40 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 3 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 45 ++++++++++++++++++--
pom.xml | 1 +
3 files changed, 44 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07c7020/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 03fe34a..2a6f16b 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -64,7 +64,8 @@ public class BuildIIWithEngineTest {
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
- protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+// protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[]{"test_kylin_ii_left_join"};
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07c7020/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index c7c07e3..c892b5f 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
@@ -45,12 +46,15 @@ import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.streaming.Stream;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
@@ -60,7 +64,9 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.text.SimpleDateFormat;
import java.util.List;
+import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.LinkedBlockingDeque;
@@ -119,11 +125,35 @@ public class BuildIIWithStreamTest {
return intermediateTableDesc.getTableName(uuid);
}
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance);
+ return segment;
+ }
+
@Test
- public void test() throws IOException, InterruptedException {
+ public void test() throws Exception {
final IIDesc desc = iiManager.getII(II_NAME).getDescriptor();
-// final String tableName = createIntermediateTable(desc, kylinConfig);
- final String tableName = "kylin_intermediate_ii_test_kylin_ii_inner_join_desc_f24b8e1f_1c1f_4835_8d78_8f21ce79a536";
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+// final String tableName = "kylin_intermediate_ii_test_kylin_ii_inner_join_desc_f24b8e1f_1c1f_4835_8d78_8f21ce79a536";
final Configuration conf = new Configuration();
HCatInputFormat.setInput(conf, "default", tableName);
final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
@@ -138,7 +168,12 @@ public class BuildIIWithStreamTest {
}
}
LinkedBlockingDeque<Stream> queue = new LinkedBlockingDeque<Stream>();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, "test_htable", desc, 0);
+ final IISegment segment = createSegment(II_NAME);
+ String[] args = new String[]{"-iiname", II_NAME, "-htablename", segment.getStorageLocationIdentifier()};
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, segment.getStorageLocationIdentifier(), desc, 0);
final Thread thread = new Thread(streamBuilder);
thread.start();
while (reader.next()) {
@@ -147,6 +182,8 @@ public class BuildIIWithStreamTest {
queue.put(new Stream(-1, null));
thread.join();
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+
}
private Stream parse(String[] row) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07c7020/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e99f77d..94f1ca3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -615,6 +615,7 @@
<excludes>
<exclude>**/BuildCubeWithEngineTest.java</exclude>
<exclude>**/BuildIIWithEngineTest.java</exclude>
+ <exclude>**/BuildIIWithStreamTest.java</exclude>
<exclude>**/SampleCubeSetupTest.java</exclude>
<exclude>**/KylinQueryTest.java</exclude>
<exclude>**/Kafka*Test.java</exclude>
[04/50] [abbrv] incubator-kylin git commit: Merge remote-tracking
branch 'origin/staging' into streaming-localdict
Posted by li...@apache.org.
Merge remote-tracking branch 'origin/staging' into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/132b0ca7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/132b0ca7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/132b0ca7
Branch: refs/heads/streaming-localdict
Commit: 132b0ca7e179a59b45899a0f77f7891321689236
Parents: 3cc984f 8cca80d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Mar 10 16:56:33 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Mar 10 16:56:33 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
9 files changed, 104 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
[40/50] [abbrv] incubator-kylin git commit: refactor meta
Posted by li...@apache.org.
refactor meta
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c1a56ddd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c1a56ddd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c1a56ddd
Branch: refs/heads/streaming-localdict
Commit: c1a56dddcb241325bbc61c9a902601dd09aafee0
Parents: 54b4066
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 13:51:45 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 13:51:45 2015 +0800
----------------------------------------------------------------------
.../invertedindex_desc/test_kylin_ii_inner_join_desc.json | 6 ++----
.../invertedindex_desc/test_kylin_ii_left_join_desc.json | 6 ++----
.../org/apache/kylin/invertedindex/index/SliceBuilder.java | 8 ++++----
3 files changed, 8 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c1a56ddd/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
index 18958b6..c78dfad 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
@@ -4,7 +4,7 @@
"name": "test_kylin_ii_inner_join_desc",
"model_name": "test_kylin_inner_join_model_desc",
"timestamp_dimension": "cal_dt",
- "bitmap_dimensions": [
+ "value_dimensions": [
{
"table": "default.test_kylin_fact",
"columns": [
@@ -41,9 +41,7 @@
"seller_type_cd",
"seller_type_desc"
]
- }
- ],
- "value_dimensions": [
+ },
{
"table": "edw.test_cal_dt",
"columns": [
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c1a56ddd/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
index 654cd38..d3b4ad7 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
@@ -4,7 +4,7 @@
"name": "test_kylin_ii_left_join_desc",
"model_name": "test_kylin_left_join_model_desc",
"timestamp_dimension": "cal_dt",
- "bitmap_dimensions": [
+ "value_dimensions": [
{
"table": "default.test_kylin_fact",
"columns": [
@@ -41,9 +41,7 @@
"seller_type_cd",
"seller_type_desc"
]
- }
- ],
- "value_dimensions": [
+ },
{
"table": "edw.test_cal_dt",
"columns": [
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c1a56ddd/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index 07cabf1..a7c79d1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -61,10 +61,10 @@ public class SliceBuilder {
// reset for next slice
nRecords = 0;
containers = new ColumnValueContainer[nColumns];
- for (int i : info.getDescriptor().getBitmapColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i,
- nRecordsCap);
- }
+// for (int i : info.getDescriptor().getBitmapColumns()) {
+// containers[i] = new CompressedValueContainer(info.getDigest(), i,
+// nRecordsCap);
+// }
for (int i : info.getDescriptor().getValueColumns()) {
containers[i] = new CompressedValueContainer(info.getDigest(), i,
nRecordsCap);
[49/50] [abbrv] incubator-kylin git commit: fix ci
Posted by li...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/effd8f8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/effd8f8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/effd8f8d
Branch: refs/heads/streaming-localdict
Commit: effd8f8d87fe57873a5473f87082c58356adcc2c
Parents: 5059723
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 21:43:10 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 21:43:10 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/BuildIIWithStreamTest.java | 17 ++++++++++++++---
.../endpoint/HbaseServerKVIterator.java | 6 ------
.../org/apache/kylin/streaming/StreamBuilder.java | 3 ++-
3 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/effd8f8d/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index c916bd8..014fd8b 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -71,8 +71,13 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.TimeZone;
import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingDeque;
+import static org.junit.Assert.fail;
+
/**
* Created by qianzhou on 3/9/15.
*/
@@ -196,6 +201,7 @@ public class BuildIIWithStreamTest {
private void buildII(String iiName) throws Exception {
final IIDesc desc = iiManager.getII(iiName).getDescriptor();
final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
final Configuration conf = new Configuration();
HCatInputFormat.setInput(conf, "default", tableName);
final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
@@ -215,14 +221,19 @@ public class BuildIIWithStreamTest {
ToolRunner.run(new IICreateHTableJob(), args);
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, segment.getStorageLocationIdentifier(), desc, 0);
- final Thread thread = new Thread(streamBuilder);
- thread.start();
while (reader.next()) {
queue.put(parse(reader.getRow()));
}
queue.put(new Stream(-1, null));
- thread.join();
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/effd8f8d/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index d0cfeb0..a7b67d8 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -40,7 +40,6 @@ import java.util.List;
*/
public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
- private TableRecordInfoDigest digest;
private RegionScanner innerScanner;
private Logger logger = LoggerFactory.getLogger(HbaseServerKVIterator.class);
@@ -48,11 +47,6 @@ public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
this.innerScanner = innerScanner;
}
- public HbaseServerKVIterator(RegionScanner innerScanner, TableRecordInfoDigest digest) {
- this.innerScanner = innerScanner;
- this.digest = digest;
- }
-
@Override
public void close() throws IOException {
IOUtils.closeQuietly(this.innerScanner);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/effd8f8d/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
index 39c3966..90aab25 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
@@ -103,8 +103,9 @@ public abstract class StreamBuilder implements Runnable {
continue;
}
}
- } catch (Exception e) {
+ } catch (Exception e) {
logger.error("build stream error, stop building", e);
+ throw new RuntimeException("build stream error, stop building", e);
}
}
}
[15/50] [abbrv] incubator-kylin git commit: #KYLIN-627, #KYLIN_628,
update binary package with bug fixes
Posted by li...@apache.org.
#KYLIN-627, #KYLIN_628, update binary package with bug fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/83fc04ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/83fc04ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/83fc04ba
Branch: refs/heads/streaming-localdict
Commit: 83fc04ba4da161b9e33aa6e8df9651f415498d2c
Parents: 59a1d66
Author: lukehan <lu...@apache.org>
Authored: Tue Mar 10 09:51:32 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Tue Mar 10 09:51:32 2015 -0700
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/83fc04ba/docs/website/download/index.md
----------------------------------------------------------------------
diff --git a/docs/website/download/index.md b/docs/website/download/index.md
index 170557f..4fc10c8 100644
--- a/docs/website/download/index.md
+++ b/docs/website/download/index.md
@@ -15,7 +15,7 @@ title: Community
<p class="big_text">Binary Package (v0.7.x)</p>
<p><a href="http://kylin.incubator.apache.org/download/kylin-0.7.1-SNAPSHOT.tgz" target="_blank">v0.7.1-SNAPSHOT</a>
- <br/>Updated Date - March 6, 2015 <br/>
+ <br/>Updated Date - March 9, 2015 <br/>
</p>
</div>
[06/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bb7bd843
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bb7bd843
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bb7bd843
Branch: refs/heads/streaming-localdict
Commit: bb7bd843a523e009fb2fbadec4ea3dd5100d2041
Parents: 2209fde 132b0ca
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Mar 10 16:57:34 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Mar 10 16:57:34 2015 +0800
----------------------------------------------------------------------
.../test_kylin_ii_inner_join_desc.json | 4 +-
.../test_kylin_ii_left_join_desc.json | 4 +-
.../invertedindex/index/TableRecordInfo.java | 1 +
.../kylin/invertedindex/model/IIDesc.java | 2 +
job/pom.xml | 6 +
.../kylin/job/execution/AbstractExecutable.java | 10 +-
.../apache/kylin/streaming/StreamBuilder.java | 41 ++++--
.../invertedindex/IIStreamBuilder.java | 43 +++---
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
17 files changed, 177 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
[13/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2be4e32e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2be4e32e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2be4e32e
Branch: refs/heads/streaming-localdict
Commit: 2be4e32ec51f4748020f81801ca2d6ac1adc112f
Parents: af6372d bb7bd84
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Mar 10 18:25:41 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Mar 10 18:25:41 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/cube/RangeKeyDistributionReducer.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[19/50] [abbrv] incubator-kylin git commit: Add a log message when a table wasn’t loaded successfully
Posted by li...@apache.org.
Add a log message when a table wasn’t loaded successfully
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/54dc31f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/54dc31f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/54dc31f7
Branch: refs/heads/streaming-localdict
Commit: 54dc31f72af63bc37deb5cd60261f880c8de2b18
Parents: 2bbaee6
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 11:32:07 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 11:32:07 2015 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/metadata/MetadataManager.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/54dc31f7/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 97d5d51..aedefd4 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -257,6 +257,7 @@ public class MetadataManager {
ResourceStore store = getStore();
TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER);
if (t == null) {
+ logger.error("Didn't load table at " + path);
return null;
}
t.init();
[25/50] [abbrv] incubator-kylin git commit: refactor test
Posted by li...@apache.org.
refactor test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7d119e9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7d119e9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7d119e9b
Branch: refs/heads/streaming-localdict
Commit: 7d119e9b68cd73b184adf6a60c4536a964e8f21a
Parents: 0939359
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 18:17:41 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 18:17:41 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 4 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 61 +++++++++++++++++---
2 files changed, 56 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d119e9b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 2a6f16b..80c075e 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -65,7 +65,7 @@ public class BuildIIWithEngineTest {
protected ExecutableManager jobService;
// protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
- protected static final String[] TEST_II_INSTANCES = new String[]{"test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[]{};
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
@@ -106,7 +106,6 @@ public class BuildIIWithEngineTest {
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
- iiManager = IIManager.getInstance(kylinConfig);
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
if(jobService.getJob(jobId) instanceof IIJob){
@@ -114,6 +113,7 @@ public class BuildIIWithEngineTest {
}
}
+ iiManager = IIManager.getInstance(kylinConfig);
for (String iiInstance : TEST_II_INSTANCES) {
IIInstance ii = iiManager.getII(iiInstance);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d119e9b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index c892b5f..af3c8e4 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -53,11 +53,14 @@ import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.streaming.Stream;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -77,7 +80,7 @@ public class BuildIIWithStreamTest {
private static final Log logger = LogFactory.getLog(BuildIIWithStreamTest.class);
- private static final String II_NAME = "test_kylin_ii_inner_join";
+ private static final String[] II_NAME = new String[]{"test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
private IIManager iiManager;
private KylinConfig kylinConfig;
@@ -94,6 +97,46 @@ public class BuildIIWithStreamTest {
kylinConfig = KylinConfig.getInstanceFromEnv();
iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+ for (String iiInstance : II_NAME) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii);
+ }
+ }
+ backup();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = {"--delete", "true"};
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
}
private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
@@ -149,11 +192,9 @@ public class BuildIIWithStreamTest {
return segment;
}
- @Test
- public void test() throws Exception {
- final IIDesc desc = iiManager.getII(II_NAME).getDescriptor();
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
final String tableName = createIntermediateTable(desc, kylinConfig);
-// final String tableName = "kylin_intermediate_ii_test_kylin_ii_inner_join_desc_f24b8e1f_1c1f_4835_8d78_8f21ce79a536";
final Configuration conf = new Configuration();
HCatInputFormat.setInput(conf, "default", tableName);
final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
@@ -168,8 +209,8 @@ public class BuildIIWithStreamTest {
}
}
LinkedBlockingDeque<Stream> queue = new LinkedBlockingDeque<Stream>();
- final IISegment segment = createSegment(II_NAME);
- String[] args = new String[]{"-iiname", II_NAME, "-htablename", segment.getStorageLocationIdentifier()};
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[]{"-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier()};
ToolRunner.run(new IICreateHTableJob(), args);
@@ -183,7 +224,13 @@ public class BuildIIWithStreamTest {
thread.join();
logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ }
}
private Stream parse(String[] row) {
[18/50] [abbrv] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2bbaee6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2bbaee6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2bbaee6c
Branch: refs/heads/streaming-localdict
Commit: 2bbaee6c2a665dfd6b02dfde3758890e76a82e5b
Parents: a5821c8 83fc04b
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 11:22:04 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 11:22:04 2015 +0800
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 +-
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
16 files changed, 120 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2bbaee6c/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
[45/50] [abbrv] incubator-kylin git commit: fix ci
Posted by li...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/afd35a08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/afd35a08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/afd35a08
Branch: refs/heads/streaming-localdict
Commit: afd35a0883a2bc86c13de8852c5cfa9e0fe007ee
Parents: 08d8e32
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 17:39:07 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 17:39:07 2015 +0800
----------------------------------------------------------------------
.../hbase/coprocessor/endpoint/EndpointAggregationTest.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/afd35a08/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
index 38c4cf0..e90f5b5 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.storage.hbase.coprocessor.FilterDecorator;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
@@ -48,6 +49,7 @@ import java.util.*;
*
* ii test
*/
+@Ignore("need to mock up TableRecordInfo")
public class EndpointAggregationTest extends LocalFileMetadataTestCase {
IIInstance ii;
TableRecordInfo tableRecordInfo;
[07/50] [abbrv] incubator-kylin git commit: config right authorize
url Kylin-570
Posted by li...@apache.org.
config right authorize url Kylin-570
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2e981533
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2e981533
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2e981533
Branch: refs/heads/streaming-localdict
Commit: 2e981533a024d4b9e5ee7f1d8132ed5ca16b5563
Parents: 7dae30e
Author: jiazhong <ji...@ebay.com>
Authored: Tue Mar 10 17:17:52 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Mar 10 17:17:52 2015 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 --
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/listeners.js | 7 -------
4 files changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index af1b83b..518a9bd 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -12,7 +12,7 @@
<scr:http auto-config="true" use-expressions="true">
<scr:http-basic entry-point-ref="unauthorisedEntryPoint" />
- <scr:intercept-url pattern="/api/user/authenticate*/**" access="permitAll" />
+ <scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll" />
<scr:intercept-url pattern="/api/query*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/metadata*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/**/metrics" access="permitAll" />
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/webapp/app/js/controllers/auth.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/auth.js b/webapp/app/js/controllers/auth.js
index 376d1b1..f7fbfea 100644
--- a/webapp/app/js/controllers/auth.js
+++ b/webapp/app/js/controllers/auth.js
@@ -43,7 +43,5 @@ KylinApp.controller('LoginCtrl', function ($scope, $rootScope, $location, $base6
$scope.loading = false;
$scope.error = "Unable to login, please check your username/password.";
});
-
- console.debug("Login event requested.");
};
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/webapp/app/js/controllers/cube.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js
index 689496a..607706d 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, CubeService, TableService, CubeGraphService, UserService, AuthenticationService,SweetAlert) {
+KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, CubeService, TableService, CubeGraphService, UserService,SweetAlert) {
$scope.newAccess = null;
$scope.state = {jsonEdit: false};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/webapp/app/js/listeners.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/listeners.js b/webapp/app/js/listeners.js
index 45f886a..4524696 100644
--- a/webapp/app/js/listeners.js
+++ b/webapp/app/js/listeners.js
@@ -92,11 +92,4 @@ KylinApp.run(function ($rootScope,$http, $location, UserService, AuthenticationS
});
- /**
- * On 'event:error', resend all the 500 requests.
- */
-// $rootScope.$on('event:error', function (event, message) {
-// var msg = !!(message) ? message : 'Failed to take action.';
-// SweetAlert.swal('Oops...', 'Action Failed: ' + msg, 'error');
-// });
});
[33/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bb673625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bb673625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bb673625
Branch: refs/heads/streaming-localdict
Commit: bb6736258fd9146b9f7fb04745e1ad1990738193
Parents: ab458df
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 18:25:11 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 18:25:11 2015 +0800
----------------------------------------------------------------------
.../index/CompressedValueContainer.java | 8 ++++++--
.../invertedindex/model/IIKeyValueCodec.java | 18 +++++++++++++++++-
.../storage/hbase/HBaseClientKVIterator.java | 2 +-
3 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
index 7a65489..7bd2266 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
@@ -42,8 +42,12 @@ public class CompressedValueContainer implements ColumnValueContainer {
byte[] compressed;
public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) {
- this.valueLen = digest.length(col);
- this.nValues = digest.getMaxID(col) + 1;
+ this(digest.length(col), digest.getMaxID(col) + 1, cap);
+ }
+
+ public CompressedValueContainer(int valueLen, int nValues, int cap) {
+ this.valueLen = valueLen;
+ this.nValues = nValues;
this.cap = cap;
this.size = 0;
this.uncompressed = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 2ebd53d..6955229 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -22,12 +22,14 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.index.ColumnValueContainer;
import org.apache.kylin.invertedindex.index.CompressedValueContainer;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import java.io.*;
import java.util.*;
/**
@@ -137,6 +139,18 @@ public class IIKeyValueCodec implements KeyValueCodec {
this.iterator = kvs.iterator();
}
+ private Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
+ try {
+ final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
+ final String type = dataInputStream.readUTF();
+ final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
+ dictionary.readFields(dataInputStream);
+ return dictionary;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private void goToNext() {
if (slice != null) { // was not fetched
return;
@@ -148,7 +162,9 @@ public class IIKeyValueCodec implements KeyValueCodec {
ImmutableBytesWritable k = kv.getKey();
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
-// localDictionaries.put(curCol, kv.getDictionary());
+ final Dictionary<?> dictionary = deserialize(kv.getDictionary());
+ addContainer(curCol, new CompressedValueContainer(dictionary.getSizeOfId(), (dictionary.getMaxId() - dictionary.getMinId() + 1), (dictionary.getMaxId() - dictionary.getMinId() + 1)));
+ localDictionaries.put(curCol, dictionary);
if (curShard != lastShard
|| curSliceTimestamp != lastSliceTimestamp) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb673625/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 05d2421..6c1085e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -67,7 +67,7 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
- IIRow pair = new IIRow(key, value);
+ IIRow pair = new IIRow(key, value, null);
@Override
public boolean hasNext() {
[41/50] [abbrv] incubator-kylin git commit: fix NPE
Posted by li...@apache.org.
fix NPE
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ea0a6236
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ea0a6236
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ea0a6236
Branch: refs/heads/streaming-localdict
Commit: ea0a6236827237779aefe370ca51864da09c433e
Parents: c1a56dd
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 14:58:15 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 14:58:15 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/invertedindex/model/IIDesc.java | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ea0a6236/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index 142ac39..cda3c4d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -20,10 +20,7 @@ package org.apache.kylin.invertedindex.model;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
import org.apache.commons.net.util.Base64;
import org.apache.hadoop.hbase.util.Bytes;
@@ -68,7 +65,7 @@ public class IIDesc extends RootPersistentEntity {
@JsonProperty("timestamp_dimension")
private String timestampDimension;
@JsonProperty("bitmap_dimensions")
- private List<DimensionDesc> bitmapDimensions;
+ private List<DimensionDesc> bitmapDimensions = Collections.emptyList();
@JsonProperty("value_dimensions")
private List<DimensionDesc> valueDimensions;
@JsonProperty("metrics")
[22/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/63695e4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/63695e4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/63695e4d
Branch: refs/heads/streaming-localdict
Commit: 63695e4d481a509b2366e81e644ac81847098b13
Parents: 66902d0 6ecd73e
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 15:41:08 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 15:41:08 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/filter/CaseTupleFilter.java | 9 ++++--
.../metadata/filter/ColumnTupleFilter.java | 8 ++++--
.../metadata/filter/CompareTupleFilter.java | 6 +++-
.../metadata/filter/ConstantTupleFilter.java | 7 +++--
.../metadata/filter/DynamicTupleFilter.java | 8 ++++--
.../metadata/filter/ExtractTupleFilter.java | 9 ++++--
.../kylin/metadata/filter/ICodeSystem.java | 23 ---------------
.../metadata/filter/IEvaluatableTuple.java | 9 ------
.../metadata/filter/LogicalTupleFilter.java | 15 ++++++----
.../kylin/metadata/filter/StringCodeSystem.java | 25 +++++++++++-----
.../kylin/metadata/filter/TupleFilter.java | 8 ++++--
.../metadata/filter/TupleFilterSerializer.java | 11 +++----
.../kylin/metadata/tuple/ICodeSystem.java | 30 ++++++++++++++++++++
.../kylin/metadata/tuple/IEvaluatableTuple.java | 9 ++++++
.../org/apache/kylin/metadata/tuple/ITuple.java | 3 +-
.../storage/hbase/DerivedFilterTranslator.java | 2 +-
.../hbase/coprocessor/CoprocessorFilter.java | 2 +-
.../hbase/coprocessor/DictCodeSystem.java | 22 ++++++++++----
.../coprocessor/observer/ObserverTuple.java | 2 +-
.../kylin/storage/filter/FilterBaseTest.java | 1 +
20 files changed, 130 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
[16/50] [abbrv] incubator-kylin git commit: KYLIN-625,
formalize ICodeSystem
Posted by li...@apache.org.
KYLIN-625, formalize ICodeSystem
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6ecd73ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6ecd73ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6ecd73ea
Branch: refs/heads/streaming-localdict
Commit: 6ecd73ea2f55b4cb39ed6ac9c9a1c33a40a19579
Parents: 2be4e32
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Mar 11 10:40:51 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Mar 11 10:40:51 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/filter/CaseTupleFilter.java | 9 ++++--
.../metadata/filter/ColumnTupleFilter.java | 8 ++++--
.../metadata/filter/CompareTupleFilter.java | 6 +++-
.../metadata/filter/ConstantTupleFilter.java | 7 +++--
.../metadata/filter/DynamicTupleFilter.java | 8 ++++--
.../metadata/filter/ExtractTupleFilter.java | 9 ++++--
.../kylin/metadata/filter/ICodeSystem.java | 23 ---------------
.../metadata/filter/IEvaluatableTuple.java | 9 ------
.../metadata/filter/LogicalTupleFilter.java | 15 ++++++----
.../kylin/metadata/filter/StringCodeSystem.java | 25 +++++++++++-----
.../kylin/metadata/filter/TupleFilter.java | 8 ++++--
.../metadata/filter/TupleFilterSerializer.java | 11 +++----
.../kylin/metadata/tuple/ICodeSystem.java | 30 ++++++++++++++++++++
.../kylin/metadata/tuple/IEvaluatableTuple.java | 9 ++++++
.../org/apache/kylin/metadata/tuple/ITuple.java | 3 +-
.../storage/hbase/DerivedFilterTranslator.java | 2 +-
.../hbase/coprocessor/CoprocessorFilter.java | 2 +-
.../hbase/coprocessor/DictCodeSystem.java | 22 ++++++++++----
.../coprocessor/observer/ObserverTuple.java | 2 +-
.../kylin/storage/filter/FilterBaseTest.java | 1 +
20 files changed, 130 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index d3ec3e3..4b85172 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -23,6 +23,9 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
/**
* @author xjiang
@@ -62,7 +65,7 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
if (whenFilters.size() != thenFilters.size()) {
elseFilter = whenFilters.remove(whenFilters.size() - 1);
}
@@ -100,12 +103,12 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(ICodeSystem<?> cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index a294112..75f534d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -28,6 +28,8 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
*
@@ -66,7 +68,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
this.tupleValue = tuple.getValue(columnRef);
return true;
}
@@ -83,7 +85,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(ICodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
@@ -103,7 +105,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
TableDesc table = null;
ColumnDesc column = new ColumnDesc();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 48b31ea..71950ff 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -27,6 +27,8 @@ import java.util.Map;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
* @author xjiang
@@ -128,6 +130,7 @@ public class CompareTupleFilter extends TupleFilter {
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
+ @SuppressWarnings({ "unchecked", "rawtypes" })
public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
// extract tuple value
Object tupleValue = null;
@@ -194,6 +197,7 @@ public class CompareTupleFilter extends TupleFilter {
return column != null && !conditionValues.isEmpty();
}
+ @SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
@@ -209,7 +213,7 @@ public class CompareTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
this.dynamicVariables.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 2280c6a..c92b743 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -24,6 +24,8 @@ import java.util.Collections;
import java.util.HashSet;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
*
@@ -63,7 +65,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
return constantValues.size() > 0;
}
@@ -77,6 +79,7 @@ public class ConstantTupleFilter extends TupleFilter {
return this.constantValues;
}
+ @SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
@@ -91,7 +94,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
this.constantValues.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index 64f12b8..3d7f65f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -23,6 +23,8 @@ import java.util.Collection;
import java.util.Collections;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
*
@@ -53,7 +55,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
return true;
}
@@ -68,7 +70,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(ICodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
BytesUtil.writeUTFString(variableName, buffer);
byte[] result = new byte[buffer.position()];
@@ -77,7 +79,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
this.variableName = BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index 929ab0f..ad1395c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -22,6 +22,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
/**
*
@@ -52,7 +55,7 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
// extract tuple value
String extractType = null;
String tupleValue = null;
@@ -112,12 +115,12 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(ICodeSystem<?> cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
deleted file mode 100644
index 39dcd16..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.apache.kylin.metadata.filter;
-
-import java.nio.ByteBuffer;
-
-/**
- * Decides how constant values in filter are coded and compared.
- *
- * TupleFilter are involved in both query engine and coprocessor. In query engine, the values are strings.
- * In coprocessor, the values are dictionary IDs.
- *
- * @author yangli9
- */
-public interface ICodeSystem {
-
- boolean isNull(Object value);
-
- int compare(Object tupleValue, Object constValue);
-
- void serialize(Object value, ByteBuffer buffer);
-
- Object deserialize(ByteBuffer buffer);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java
deleted file mode 100644
index 92f83f1..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package org.apache.kylin.metadata.filter;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-public interface IEvaluatableTuple {
-
- public Object getValue(TblColRef col);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 6b77cb5..0c8f96b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -24,6 +24,9 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
public class LogicalTupleFilter extends TupleFilter {
@@ -69,7 +72,7 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
switch (this.operator) {
case AND:
return evalAnd(tuple, cs);
@@ -82,7 +85,7 @@ public class LogicalTupleFilter extends TupleFilter {
}
}
- private boolean evalAnd(IEvaluatableTuple tuple, ICodeSystem cs) {
+ private boolean evalAnd(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
for (TupleFilter filter : this.children) {
if (!filter.evaluate(tuple, cs)) {
return false;
@@ -91,7 +94,7 @@ public class LogicalTupleFilter extends TupleFilter {
return true;
}
- private boolean evalOr(IEvaluatableTuple tuple, ICodeSystem cs) {
+ private boolean evalOr(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
for (TupleFilter filter : this.children) {
if (filter.evaluate(tuple, cs)) {
return true;
@@ -100,7 +103,7 @@ public class LogicalTupleFilter extends TupleFilter {
return false;
}
- private boolean evalNot(IEvaluatableTuple tuple, ICodeSystem cs) {
+ private boolean evalNot(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
return !this.children.get(0).evaluate(tuple, cs);
}
@@ -115,12 +118,12 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(ICodeSystem<?> cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem cs) {
+ public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
index b39b398..5240491 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
@@ -3,13 +3,14 @@ package org.apache.kylin.metadata.filter;
import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
/**
* A simple code system where all values are strings and conform to string comparison system.
*
* @author yangli9
*/
-public class StringCodeSystem implements ICodeSystem {
+public class StringCodeSystem implements ICodeSystem<String> {
public static final StringCodeSystem INSTANCE = new StringCodeSystem();
@@ -18,22 +19,32 @@ public class StringCodeSystem implements ICodeSystem {
}
@Override
- public boolean isNull(Object value) {
+ public String encode(Object value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object decode(String code) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isNull(String value) {
return value == null;
}
@Override
- public int compare(Object tupleValue, Object constValue) {
- return ((String) tupleValue).compareTo((String) constValue);
+ public int compare(String tupleValue, String constValue) {
+ return tupleValue.compareTo(constValue);
}
@Override
- public void serialize(Object value, ByteBuffer buffer) {
- BytesUtil.writeUTFString((String) value, buffer);
+ public void serialize(String value, ByteBuffer buffer) {
+ BytesUtil.writeUTFString( value, buffer);
}
@Override
- public Object deserialize(ByteBuffer buffer) {
+ public String deserialize(ByteBuffer buffer) {
return BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 5293d42..75d55c3 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -25,6 +25,8 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import com.google.common.collect.Maps;
@@ -188,13 +190,13 @@ public abstract class TupleFilter {
public abstract boolean isEvaluable();
- public abstract boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs);
+ public abstract boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs);
public abstract Collection<?> getValues();
- abstract byte[] serialize(ICodeSystem cs);
+ abstract byte[] serialize(ICodeSystem<?> cs);
- abstract void deserialize(byte[] bytes, ICodeSystem cs);
+ abstract void deserialize(byte[] bytes, ICodeSystem<?> cs);
public static boolean isEvaluableRecursively(TupleFilter filter) {
if (filter == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index ba4b5e6..5164c5b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Stack;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
/**
* http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
@@ -46,11 +47,11 @@ public class TupleFilterSerializer {
}
}
- public static byte[] serialize(TupleFilter rootFilter, ICodeSystem cs) {
+ public static byte[] serialize(TupleFilter rootFilter, ICodeSystem<?> cs) {
return serialize(rootFilter, null, cs);
}
- public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, ICodeSystem cs) {
+ public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, ICodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
internalSerialize(rootFilter, decorator, buffer, cs);
byte[] result = new byte[buffer.position()];
@@ -58,7 +59,7 @@ public class TupleFilterSerializer {
return result;
}
- private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem cs) {
+ private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem<?> cs) {
if (decorator != null) { // give decorator a chance to manipulate the
// output filter
filter = decorator.onSerialize(filter);
@@ -83,7 +84,7 @@ public class TupleFilterSerializer {
}
}
- private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem cs) {
+ private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem<?> cs) {
if (flag < 0) {
BytesUtil.writeVInt(-1, buffer);
} else {
@@ -95,7 +96,7 @@ public class TupleFilterSerializer {
}
}
- public static TupleFilter deserialize(byte[] bytes, ICodeSystem cs) {
+ public static TupleFilter deserialize(byte[] bytes, ICodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
TupleFilter rootFilter = null;
Stack<TupleFilter> parentStack = new Stack<TupleFilter>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java
new file mode 100644
index 0000000..b5bc025
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java
@@ -0,0 +1,30 @@
+package org.apache.kylin.metadata.tuple;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Decides how constant values are coded and compared.
+ *
+ * TupleFilter are involved in both query engine and coprocessor. In query engine, the values are strings.
+ * In coprocessor, the values are dictionary IDs.
+ *
+ * The type parameter is java type of code, which should be bytes. However some legacy implementation
+ * stores code as String.
+ *
+ * @author yangli9
+ */
+public interface ICodeSystem<T> {
+
+ T encode(Object value);
+
+ Object decode(T code);
+
+ boolean isNull(T code);
+
+ int compare(T code1, T code2);
+
+ void serialize(T code, ByteBuffer buffer);
+
+ T deserialize(ByteBuffer code);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
new file mode 100644
index 0000000..ce957b7
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
@@ -0,0 +1,9 @@
+package org.apache.kylin.metadata.tuple;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+public interface IEvaluatableTuple {
+
+ public Object getValue(TblColRef col);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
index e5dbf08..05fe563 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
@@ -20,7 +20,6 @@ package org.apache.kylin.metadata.tuple;
import java.util.List;
-import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -37,7 +36,7 @@ public interface ITuple extends IEvaluatableTuple {
public Object[] getAllValues();
- // declared in IEvaluatableTuple: public Object getValue(TblColRef col);
+ // declared from IEvaluatableTuple: public Object getValue(TblColRef col);
public Object getValue(String field);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index 7fc0429..36a281f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -30,12 +30,12 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index ea4b2e1..a91999f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -22,10 +22,10 @@ import java.util.Set;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
index 14497f4..cada02b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
@@ -4,9 +4,9 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.filter.ICodeSystem;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
-public class DictCodeSystem implements ICodeSystem {
+public class DictCodeSystem implements ICodeSystem<String> {
public static final DictCodeSystem INSTANCE = new DictCodeSystem();
@@ -15,7 +15,17 @@ public class DictCodeSystem implements ICodeSystem {
}
@Override
- public boolean isNull(Object value) {
+ public String encode(Object value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object decode(String code) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isNull(String value) {
if (value == null)
return true;
@@ -28,17 +38,17 @@ public class DictCodeSystem implements ICodeSystem {
}
@Override
- public int compare(Object tupleValue, Object constValue) {
+ public int compare(String tupleValue, String constValue) {
return ((String) tupleValue).compareTo((String) constValue);
}
@Override
- public void serialize(Object value, ByteBuffer buffer) {
+ public void serialize(String value, ByteBuffer buffer) {
BytesUtil.writeUTFString((String) value, buffer);
}
@Override
- public Object deserialize(ByteBuffer buffer) {
+ public String deserialize(ByteBuffer buffer) {
return BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
index 288afa9..cdbe392 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
@@ -20,8 +20,8 @@ package org.apache.kylin.storage.hbase.coprocessor.observer;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ecd73ea/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
index 97e0e33..fe9d336 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
@@ -27,6 +27,7 @@ import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
[03/50] [abbrv] incubator-kylin git commit: smaller region size
Posted by li...@apache.org.
smaller region size
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/37f9a735
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/37f9a735
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/37f9a735
Branch: refs/heads/streaming-localdict
Commit: 37f9a735cd8d5ca27b208182117d844c85bf1c70
Parents: 6a91c5c
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Mar 10 16:56:27 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Mar 10 16:56:27 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/cube/RangeKeyDistributionReducer.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37f9a735/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
index 6bdacfd..6da179c 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
@@ -39,11 +39,11 @@ import org.apache.kylin.job.constant.BatchConstants;
public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable, Text, LongWritable> {
public static final long ONE_GIGA_BYTES = 1024L * 1024L * 1024L;
- public static final int SMALL_CUT = 10; // 10 GB per region
- public static final int MEDIUM_CUT = 20; // 20 GB per region
- public static final int LARGE_CUT = 100; // 100 GB per region
+ public static final int SMALL_CUT = 5; // 5 GB per region
+ public static final int MEDIUM_CUT = 10; // 10 GB per region
+ public static final int LARGE_CUT = 50; // 50 GB per region
- public static final int MAX_REGION = 500;
+ public static final int MAX_REGION = 1000;
private static final Logger logger = LoggerFactory.getLogger(RangeKeyDistributionReducer.class);
[43/50] [abbrv] incubator-kylin git commit: fix
Posted by li...@apache.org.
fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5880b79b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5880b79b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5880b79b
Branch: refs/heads/streaming-localdict
Commit: 5880b79bd99d9be52e22022de0b75d0fd4bda8d4
Parents: 73c53b4
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 16:51:33 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 16:51:33 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/TableRecordInfo.java | 65 ++++++++++----------
.../index/TableRecordInfoDigest.java | 14 ++++-
.../invertedindex/model/IIKeyValueCodec.java | 30 ++++++---
.../storage/hbase/InvertedIndexHBaseTest.java | 26 +++-----
4 files changed, 77 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5880b79b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index c178742..3f8b72a 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -18,20 +18,19 @@
package org.apache.kylin.invertedindex.index;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
import com.google.common.collect.Maps;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
/**
* @author yangli9
* <p/>
@@ -46,31 +45,7 @@ public class TableRecordInfo {
final Map<Integer, Dictionary<?>> dictionaryMap;
public TableRecordInfo(IISegment iiSegment) {
-
- this.desc = iiSegment.getIIInstance().getDescriptor();
- this.dictionaryMap = Maps.newHashMap();
- Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap = Maps.newHashMap();
-
- DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig());
- int index = 0;
- for (TblColRef tblColRef : desc.listAllColumns()) {
- ColumnDesc col = tblColRef.getColumn();
- if (desc.isMetricsCol(index)) {
- measureCodecMap.put(tblColRef, FixedLenMeasureCodec.get(col.getType()));
- } else {
- String dictPath = iiSegment.getDictResPath(tblColRef);
- if (dictPath != null) {
- try {
- dictionaryMap.put(index, dictMgr.getDictionary(dictPath));
- } catch (IOException e) {
- throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
- }
- }
- }
- index++;
- }
-
- digest = createDigest(dictionaryMap, measureCodecMap);
+ this(iiSegment.getIIDesc(), Collections.<Integer, Dictionary<?>>emptyMap());
}
public TableRecordInfo(IIDesc desc, Map<Integer, Dictionary<?>> dictionaryMap) {
@@ -127,6 +102,34 @@ public class TableRecordInfo {
return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
}
+ public static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Map<Integer, Dictionary<?>> dictionaryMap) {
+ int[] dictMaxIds = new int[nColumns];
+ int[] lengths = new int[nColumns];
+ for (int i = 0; i < nColumns; ++i) {
+ if (isMetric[i]) {
+ final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
+ lengths[i] = fixedLenMeasureCodec.getLength();
+ } else {
+ final Dictionary<?> dictionary = dictionaryMap.get(i);
+ if (dictionary != null) {
+ lengths[i] = dictionary.getSizeOfId();
+ dictMaxIds[i] = dictionary.getMaxId();
+ }
+ }
+ }
+ // offsets
+ int pos = 0;
+ int[] offsets = new int[nColumns];
+ for (int i = 0; i < nColumns; i++) {
+ offsets[i] = pos;
+ pos += lengths[i];
+ }
+
+ int byteFormLen = pos;
+
+ return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
+ }
+
public TableRecord createTableRecord() {
return new TableRecord(digest.createTableRecordBytes(), this);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5880b79b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index ff6b192..cf519d7 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -89,7 +89,19 @@ public class TableRecordInfoDigest {
return ret;
}
- public RawTableRecord createTableRecordBytes() {
+ public boolean[] getIsMetric() {
+ return isMetric;
+ }
+
+ public int[] getLengths() {
+ return lengths;
+ }
+
+ public String[] getMetricDataTypes() {
+ return metricDataTypes;
+ }
+
+ public RawTableRecord createTableRecordBytes() {
return new RawTableRecord(this);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5880b79b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index ad1b25c..e3284fc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -115,16 +115,16 @@ public class IIKeyValueCodec implements KeyValueCodec {
@Override
public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
return new IIRowDecoder(digest, kvs.iterator());
-// return new Decoder(kvs, digest);
+// return new Decoder(kvs, incompleteDigest);
}
private static class IIRowDecoder implements Iterable<Slice> {
- private final TableRecordInfoDigest digest;
+ private final TableRecordInfoDigest incompleteDigest;
private final Iterator<IIRow> iterator;
private IIRowDecoder(TableRecordInfoDigest digest, Iterator<IIRow> iterator) {
- this.digest = digest;
+ this.incompleteDigest = digest;
this.iterator = iterator;
}
@@ -139,14 +139,15 @@ public class IIKeyValueCodec implements KeyValueCodec {
@Override
public Slice next() {
int columns = 0;
- ColumnValueContainer[] valueContainers = new ColumnValueContainer[digest.getColumnCount()];
+ ColumnValueContainer[] valueContainers = new ColumnValueContainer[incompleteDigest.getColumnCount()];
Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
boolean firstTime = true;
short curShard = 0;
long curTimestamp = 0;
short lastShard = 0;
long lastTimestamp = 0;
- while (iterator().hasNext() && columns < digest.getColumnCount()) {
+
+ while (iterator().hasNext() && columns < incompleteDigest.getColumnCount()) {
final IIRow row = iterator.next();
final ImmutableBytesWritable key = row.getKey();
int i = key.getOffset();
@@ -161,16 +162,25 @@ public class IIKeyValueCodec implements KeyValueCodec {
}
int curCol = BytesUtil.readUnsigned(key.get(), i, COLNO_LEN);
- CompressedValueContainer c = new CompressedValueContainer(digest, curCol, 0);
- c.fromBytes(row.getValue());
- valueContainers[curCol] = c;
- localDictionaries.put(curCol, deserialize(row.getDictionary()));
+ final Dictionary<?> dictionary = deserialize(row.getDictionary());
+ if (incompleteDigest.isMetrics(curCol)) {
+ CompressedValueContainer c = new CompressedValueContainer(incompleteDigest, curCol, 0);
+ c.fromBytes(row.getValue());
+ valueContainers[curCol] = c;
+ } else {
+ CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), dictionary.getMaxId() - dictionary.getMinId() + 1, 0);
+ c.fromBytes(row.getValue());
+ valueContainers[curCol] = c;
+ }
+ localDictionaries.put(curCol, dictionary);
columns++;
lastShard = curShard;
lastTimestamp = curTimestamp;
firstTime = false;
}
- Preconditions.checkArgument(columns == digest.getColumnCount(), "column count is " + columns + " should be equals to digest.getColumnCount() " + digest.getColumnCount());
+ Preconditions.checkArgument(columns == incompleteDigest.getColumnCount(), "column count is " + columns + " should be equals to incompleteDigest.getColumnCount() " + incompleteDigest.getColumnCount());
+
+ TableRecordInfoDigest digest = TableRecordInfo.createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries);
Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers);
slice.setLocalDictionaries(localDictionaries);
return slice;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5880b79b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index 9e37447..bcf5b10 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -18,23 +18,12 @@
package org.apache.kylin.storage.hbase;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
import com.google.common.collect.Lists;
-
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.invertedindex.IIInstance;
@@ -46,6 +35,11 @@ import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
/**
* @author yangli9
@@ -69,7 +63,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
hconn = HConnectionManager.createConnection(hconf);
- this.info = new TableRecordInfo(seg.getIIDesc(), null);
+ this.info = new TableRecordInfo(seg);
}
@After
[26/50] [abbrv] incubator-kylin git commit: Merge remote-tracking
branch 'origin/streaming' into streaming-localdict
Posted by li...@apache.org.
Merge remote-tracking branch 'origin/streaming' into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0b5d0134
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0b5d0134
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0b5d0134
Branch: refs/heads/streaming-localdict
Commit: 0b5d013473d9a91ef887162ba045b2f8807de397
Parents: 7d119e9 0b71528
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 18:19:46 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 18:19:46 2015 +0800
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 1 +
.../kylin/job/hadoop/AbstractHadoopJob.java | 65 ++++++++++++++------
.../apache/kylin/metadata/MetadataManager.java | 1 +
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/listeners.js | 7 ---
8 files changed, 50 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
[27/50] [abbrv] incubator-kylin git commit: ignore
BuildIIWithEngineTest
Posted by li...@apache.org.
ignore BuildIIWithEngineTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8be38e7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8be38e7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8be38e7d
Branch: refs/heads/streaming-localdict
Commit: 8be38e7d099916483204da0fea6e5bc6ded76664
Parents: 0b5d013
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 18:46:30 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 18:46:30 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/BuildIIWithEngineTest.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8be38e7d/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 80c075e..af8682e 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -38,10 +38,7 @@ import org.apache.kylin.job.invertedindex.IIJob;
import org.apache.kylin.job.invertedindex.IIJobBuilder;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
import java.io.File;
import java.io.IOException;
@@ -64,8 +61,7 @@ public class BuildIIWithEngineTest {
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
-// protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
- protected static final String[] TEST_II_INSTANCES = new String[]{};
+ protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
@@ -138,6 +134,7 @@ public class BuildIIWithEngineTest {
}
@Test
+ @Ignore
public void testBuildII() throws Exception {
String[] testCase = new String[]{"buildIIInnerJoin", "buildIILeftJoin"};
[32/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab458df4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab458df4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab458df4
Branch: refs/heads/streaming-localdict
Commit: ab458df401c23ea86dd5537e35c6d93669961246
Parents: 7b48b51
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 17:47:02 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 17:47:02 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/BitMapContainer.java | 382 +++++++++----------
.../kylin/invertedindex/index/SliceBuilder.java | 3 -
.../invertedindex/index/TableRecordInfo.java | 22 +-
.../invertedindex/model/IIKeyValueCodec.java | 164 +++-----
.../apache/kylin/invertedindex/model/IIRow.java | 67 ++++
.../invertedindex/model/KeyValueCodec.java | 49 +++
.../kylin/invertedindex/model/KeyValuePair.java | 78 ----
.../apache/kylin/invertedindex/tools/IICLI.java | 16 +-
.../invertedindex/InvertedIndexLocalTest.java | 40 +-
.../invertedindex/InvertedIndexReducer.java | 6 +-
.../storage/hbase/HBaseClientKVIterator.java | 13 +-
.../endpoint/HbaseServerKVIterator.java | 110 +++---
.../hbase/coprocessor/endpoint/IIEndpoint.java | 24 +-
.../invertedindex/IIStreamBuilder.java | 57 +--
14 files changed, 489 insertions(+), 542 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
index 066a3c8..ad3b0c1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
@@ -34,194 +34,194 @@ import org.apache.kylin.dict.Dictionary;
/**
* @author yangli9
*/
-public class BitMapContainer implements ColumnValueContainer {
-
- int valueLen;
- int nValues;
- int size;
- ConciseSet[] sets;
- boolean closedForChange;
-
- transient byte[] temp;
-
- public BitMapContainer(TableRecordInfoDigest digest, int col) {
- this.valueLen = digest.length(col);
- this.size = 0;
- this.nValues = digest.getMaxID(col) + 1;
- this.sets = null;
- this.closedForChange = false;
-
- this.temp = new byte[valueLen];
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
- append(value);
- }
-
- public void append(int value) {
- checkUpdateMode();
- if (value == Dictionary.NULL_ID[valueLen]) {
- value = nValues; // set[nValues] holds NULL
- }
- sets[value].add(size);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- int value = getValueIntAt(i);
- BytesUtil.writeUnsigned(value, temp, 0, valueLen);
- valueBytes.set(temp, 0, valueLen);
- }
-
- @Override
- public ConciseSet getBitMap(Integer startId, Integer endId) {
- if (startId == null && endId == null) {
- return sets[this.nValues];
- }
-
- int start = 0;
- int end = this.nValues - 1;
- if (startId != null) {
- start = startId;
- }
- if (endId != null) {
- end = endId;
- }
-
- ConciseSet ret = new ConciseSet();
- for (int i = start; i <= end; ++i) {
- ConciseSet temp = getBitMap(i);
- ret.addAll(temp);
- }
- return ret;
- }
-
- private ConciseSet getBitMap(int valueId) {
- if (valueId >= 0 && valueId <= getMaxValueId())
- return sets[valueId];
- else
- return sets[this.nValues];
- }
-
- @Override
- public int getMaxValueId() {
- return this.nValues - 1;
- }
-
- public int getValueIntAt(int i) {
- for (int v = 0; v < nValues; v++) {
- if (sets[v].contains(i)) {
- return v;
- }
- }
- // if v is not in [0..nValues-1], then it must be nValue (NULL)
- return Dictionary.NULL_ID[valueLen];
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalStateException();
- }
- if (sets == null) {
- sets = new ConciseSet[nValues + 1];
- for (int i = 0; i <= nValues; i++) {
- sets[i] = new ConciseSet();
- }
- }
- }
-
- private boolean isClosedForChange() {
- return closedForChange;
- }
-
- @Override
- public void closeForChange() {
- closedForChange = true;
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public List<ImmutableBytesWritable> toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
-
- List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
- for (int i = 0; i <= nValues; i++) {
- r.add(setToBytes(sets[i]));
- }
- return r;
- }
-
- public void fromBytes(List<ImmutableBytesWritable> bytes) {
- assert nValues + 1 == bytes.size();
- sets = new ConciseSet[nValues + 1];
- size = 0;
- for (int i = 0; i <= nValues; i++) {
- sets[i] = bytesToSet(bytes.get(i));
- size += sets[i].size();
- }
- closedForChange = true;
- }
-
- private ImmutableBytesWritable setToBytes(ConciseSet set) {
- byte[] array;
- if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
- // set is empty
- array = BytesUtil.EMPTY_BYTE_ARRAY;
- else
- array = set.toByteBuffer().array();
- return new ImmutableBytesWritable(array);
- }
-
- private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
- if (bytes.get() == null || bytes.getLength() == 0) {
- return new ConciseSet();
- } else {
- IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
- int[] words = new int[intBuffer.capacity()];
- intBuffer.get(words);
- return new ConciseSet(words, false);
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (closedForChange ? 1231 : 1237);
- result = prime * result + nValues;
- result = prime * result + Arrays.hashCode(sets);
- result = prime * result + size;
- result = prime * result + valueLen;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- BitMapContainer other = (BitMapContainer) obj;
- if (closedForChange != other.closedForChange)
- return false;
- if (nValues != other.nValues)
- return false;
- if (!Arrays.equals(sets, other.sets))
- return false;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- return true;
- }
-
-}
+//public class BitMapContainer implements ColumnValueContainer {
+//
+// int valueLen;
+// int nValues;
+// int size;
+// ConciseSet[] sets;
+// boolean closedForChange;
+//
+// transient byte[] temp;
+//
+// public BitMapContainer(TableRecordInfoDigest digest, int col) {
+// this.valueLen = digest.length(col);
+// this.size = 0;
+// this.nValues = digest.getMaxID(col) + 1;
+// this.sets = null;
+// this.closedForChange = false;
+//
+// this.temp = new byte[valueLen];
+// }
+//
+// @Override
+// public void append(ImmutableBytesWritable valueBytes) {
+// int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
+// append(value);
+// }
+//
+// public void append(int value) {
+// checkUpdateMode();
+// if (value == Dictionary.NULL_ID[valueLen]) {
+// value = nValues; // set[nValues] holds NULL
+// }
+// sets[value].add(size);
+// size++;
+// }
+//
+// @Override
+// public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
+// int value = getValueIntAt(i);
+// BytesUtil.writeUnsigned(value, temp, 0, valueLen);
+// valueBytes.set(temp, 0, valueLen);
+// }
+//
+// @Override
+// public ConciseSet getBitMap(Integer startId, Integer endId) {
+// if (startId == null && endId == null) {
+// return sets[this.nValues];
+// }
+//
+// int start = 0;
+// int end = this.nValues - 1;
+// if (startId != null) {
+// start = startId;
+// }
+// if (endId != null) {
+// end = endId;
+// }
+//
+// ConciseSet ret = new ConciseSet();
+// for (int i = start; i <= end; ++i) {
+// ConciseSet temp = getBitMap(i);
+// ret.addAll(temp);
+// }
+// return ret;
+// }
+//
+// private ConciseSet getBitMap(int valueId) {
+// if (valueId >= 0 && valueId <= getMaxValueId())
+// return sets[valueId];
+// else
+// return sets[this.nValues];
+// }
+//
+// @Override
+// public int getMaxValueId() {
+// return this.nValues - 1;
+// }
+//
+// public int getValueIntAt(int i) {
+// for (int v = 0; v < nValues; v++) {
+// if (sets[v].contains(i)) {
+// return v;
+// }
+// }
+// // if v is not in [0..nValues-1], then it must be nValue (NULL)
+// return Dictionary.NULL_ID[valueLen];
+// }
+//
+// private void checkUpdateMode() {
+// if (isClosedForChange()) {
+// throw new IllegalStateException();
+// }
+// if (sets == null) {
+// sets = new ConciseSet[nValues + 1];
+// for (int i = 0; i <= nValues; i++) {
+// sets[i] = new ConciseSet();
+// }
+// }
+// }
+//
+// private boolean isClosedForChange() {
+// return closedForChange;
+// }
+//
+// @Override
+// public void closeForChange() {
+// closedForChange = true;
+// }
+//
+// @Override
+// public int getSize() {
+// return size;
+// }
+//
+// public List<ImmutableBytesWritable> toBytes() {
+// if (isClosedForChange() == false)
+// closeForChange();
+//
+// List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
+// for (int i = 0; i <= nValues; i++) {
+// r.add(setToBytes(sets[i]));
+// }
+// return r;
+// }
+//
+// public void fromBytes(List<ImmutableBytesWritable> bytes) {
+// assert nValues + 1 == bytes.size();
+// sets = new ConciseSet[nValues + 1];
+// size = 0;
+// for (int i = 0; i <= nValues; i++) {
+// sets[i] = bytesToSet(bytes.get(i));
+// size += sets[i].size();
+// }
+// closedForChange = true;
+// }
+//
+// private ImmutableBytesWritable setToBytes(ConciseSet set) {
+// byte[] array;
+// if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
+// // set is empty
+// array = BytesUtil.EMPTY_BYTE_ARRAY;
+// else
+// array = set.toByteBuffer().array();
+// return new ImmutableBytesWritable(array);
+// }
+//
+// private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
+// if (bytes.get() == null || bytes.getLength() == 0) {
+// return new ConciseSet();
+// } else {
+// IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
+// int[] words = new int[intBuffer.capacity()];
+// intBuffer.get(words);
+// return new ConciseSet(words, false);
+// }
+// }
+//
+// @Override
+// public int hashCode() {
+// final int prime = 31;
+// int result = 1;
+// result = prime * result + (closedForChange ? 1231 : 1237);
+// result = prime * result + nValues;
+// result = prime * result + Arrays.hashCode(sets);
+// result = prime * result + size;
+// result = prime * result + valueLen;
+// return result;
+// }
+//
+// @Override
+// public boolean equals(Object obj) {
+// if (this == obj)
+// return true;
+// if (obj == null)
+// return false;
+// if (getClass() != obj.getClass())
+// return false;
+// BitMapContainer other = (BitMapContainer) obj;
+// if (closedForChange != other.closedForChange)
+// return false;
+// if (nValues != other.nValues)
+// return false;
+// if (!Arrays.equals(sets, other.sets))
+// return false;
+// if (size != other.size)
+// return false;
+// if (valueLen != other.valueLen)
+// return false;
+// return true;
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index e217df0..72050a1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -61,9 +61,6 @@ public class SliceBuilder {
// reset for next slice
nRecords = 0;
containers = new ColumnValueContainer[nColumns];
- for (int i : info.getDescriptor().getBitmapColumns()) {
- containers[i] = new BitMapContainer(info.getDigest(), i);
- }
for (int i : info.getDescriptor().getValueColumns()) {
containers[i] = new CompressedValueContainer(info.getDigest(), i,
nRecordsCap);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 0cfb05e..3e72f43 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -43,7 +43,7 @@ public class TableRecordInfo {
final IIDesc desc;
final TableRecordInfoDigest digest;
- final Map<TblColRef, Dictionary<?>> dictionaryMap;
+ final Map<Integer, Dictionary<?>> dictionaryMap;
public TableRecordInfo(IISegment iiSegment) {
@@ -59,10 +59,12 @@ public class TableRecordInfo {
measureCodecMap.put(tblColRef, FixedLenMeasureCodec.get(col.getType()));
} else {
String dictPath = iiSegment.getDictResPath(tblColRef);
- try {
- dictionaryMap.put(tblColRef, dictMgr.getDictionary(dictPath));
- } catch (IOException e) {
- throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
+ if (dictPath != null) {
+ try {
+ dictionaryMap.put(index, dictMgr.getDictionary(dictPath));
+ } catch (IOException e) {
+ throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
+ }
}
}
index++;
@@ -71,7 +73,7 @@ public class TableRecordInfo {
digest = createDigest(dictionaryMap, measureCodecMap);
}
- public TableRecordInfo(IIDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+ public TableRecordInfo(IIDesc desc, Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
this.desc = desc;
this.dictionaryMap = dictionaryMap;
this.digest = createDigest(dictionaryMap, measureCodecMap);
@@ -81,7 +83,7 @@ public class TableRecordInfo {
return digest;
}
- private TableRecordInfoDigest createDigest(Map<TblColRef, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+ private TableRecordInfoDigest createDigest(Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
int nColumns = getColumns().size();
boolean[] isMetric = new boolean[nColumns];
int[] lengths = new int[nColumns];
@@ -97,10 +99,10 @@ public class TableRecordInfo {
lengths[i] = fixedLenMeasureCodec.getLength();
}
} else {
- final Dictionary<?> dictionary = dictionaryMap.get(tblColRef);
+ final Dictionary<?> dictionary = dictionaryMap.get(i);
if (dictionary != null) {
lengths[i] = dictionary.getSizeOfId();
- dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+ dictMaxIds[i] = dictionary.getMaxId();
}
}
}
@@ -149,7 +151,7 @@ public class TableRecordInfo {
@SuppressWarnings("unchecked")
public Dictionary<String> dict(int col) {
// yes, all dictionaries are string based
- return (Dictionary<String>) dictionaryMap.get(desc.listAllColumns().get(col));
+ return (Dictionary<String>) dictionaryMap.get(col);
}
public int getTimestampColumn() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 8ea4e70..2ebd53d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -18,41 +18,37 @@
package org.apache.kylin.invertedindex.model;
-import java.util.*;
-
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-
-import com.google.common.collect.Lists;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.invertedindex.index.ColumnValueContainer;
+import org.apache.kylin.invertedindex.index.CompressedValueContainer;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+
+import java.util.*;
/**
* @author yangli9
*/
-public class IIKeyValueCodec {
+public class IIKeyValueCodec implements KeyValueCodec {
public static final int SHARD_LEN = 2;
public static final int TIMEPART_LEN = 8;
public static final int COLNO_LEN = 2;
- private TableRecordInfoDigest infoDigest;
-
- public IIKeyValueCodec(TableRecordInfoDigest digest) {
- this.infoDigest = digest;
+ public IIKeyValueCodec() {
}
- public Collection<KeyValuePair> encodeKeyValue(Slice slice) {
- ArrayList<KeyValuePair> result = Lists
+ @Override
+ public Collection<IIRow> encodeKeyValue(Slice slice) {
+ ArrayList<IIRow> result = Lists
.newArrayList();
ColumnValueContainer[] containers = slice.getColumnValueContainers();
for (int col = 0; col < containers.length; col++) {
- if (containers[col] instanceof BitMapContainer) {
- result.addAll(collectKeyValues(slice, col, (BitMapContainer) containers[col]));
- } else if (containers[col] instanceof CompressedValueContainer) {
+ if (containers[col] instanceof CompressedValueContainer) {
result.add(collectKeyValues(slice, col, (CompressedValueContainer) containers[col]));
} else {
throw new IllegalArgumentException("Unknown container class "
@@ -62,37 +58,21 @@ public class IIKeyValueCodec {
return result;
}
- private KeyValuePair collectKeyValues(Slice slice,
+ private IIRow collectKeyValues(Slice slice,
int col,
CompressedValueContainer container) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(),
- slice.getTimestamp(), col, -1);
+ ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col);
ImmutableBytesWritable value = container.toBytes();
- return new KeyValuePair(col, key, value);
- }
-
- private List<KeyValuePair> collectKeyValues(Slice slice,
- int col,
- BitMapContainer container) {
- List<ImmutableBytesWritable> values = container.toBytes();
- ArrayList<KeyValuePair> list = Lists.newArrayListWithExpectedSize(values.size());
- for (int v = 0; v < values.size(); v++) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(),
- slice.getTimestamp(), col, v);
- list.add(new KeyValuePair(col, key, values.get(v)));
- }
- return list;
+ return new IIRow(key, value, null);
}
- ImmutableBytesWritable encodeKey(short shard, long timestamp, int col,
- int colValue) {
+ ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) {
byte[] bytes = new byte[20];
- int len = encodeKey(shard, timestamp, col, colValue, bytes, 0);
+ int len = encodeKey(shard, timestamp, col, bytes, 0);
return new ImmutableBytesWritable(bytes, 0, len);
}
- int encodeKey(short shard, long timestamp, int col, int colValue,
- byte[] buf, int offset) {
+ int encodeKey(short shard, long timestamp, int col, byte[] buf, int offset) {
int i = offset;
BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
@@ -103,55 +83,72 @@ public class IIKeyValueCodec {
BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
i += COLNO_LEN;
- if (colValue >= 0) {
- int colLen = infoDigest.length(col);
- BytesUtil.writeUnsigned(colValue, buf, i, colLen);
- i += colLen;
- }
-
return i - offset;
}
- public Iterable<Slice> decodeKeyValue(
- Iterable<KeyValuePair> kvs) {
- return new Decoder(infoDigest, kvs);
+ @Override
+ public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
+ return new Decoder(kvs);
}
+ private static class SliceIterator implements Iterator<Slice> {
+
+ private final Iterator<IIRow> rowIterator;
+
+ public SliceIterator(Iterator<IIRow> rowIterator) {
+ this.rowIterator = rowIterator;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!rowIterator.hasNext()) {
+ return false;
+ }
+ return false;
+ }
+
+ @Override
+ public Slice next() {
+ return null;
+ }
+
+ @Override
+ public void remove() {
+
+ }
+ }
+
private static class Decoder implements Iterable<Slice> {
TableRecordInfoDigest info;
- Iterator<KeyValuePair> iterator;
+ Iterator<IIRow> iterator;
- Slice next = null;
+ Slice slice = null;
short curShard = Short.MIN_VALUE;
long curSliceTimestamp = Long.MIN_VALUE;
int curCol = -1;
- int curColValue = -1;
short lastShard = Short.MIN_VALUE;
long lastSliceTimestamp = Long.MIN_VALUE;
int lastCol = -1;
ColumnValueContainer[] containers = null;
- List<ImmutableBytesWritable> bitMapValues = Lists.newArrayList();
Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
- Decoder(TableRecordInfoDigest info,
- Iterable<KeyValuePair> kvs) {
- this.info = info;
+ Decoder(Iterable<IIRow> kvs) {
this.iterator = kvs.iterator();
}
private void goToNext() {
- if (next != null) { // was not fetched
+ if (slice != null) { // was not fetched
return;
}
// NOTE the input keys are ordered
- while (next == null && iterator.hasNext()) {
- KeyValuePair kv = iterator.next();
+ while (slice == null && iterator.hasNext()) {
+ IIRow kv = iterator.next();
ImmutableBytesWritable k = kv.getKey();
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
- localDictionaries.put(curCol, kv.getDictionary());
+// localDictionaries.put(curCol, kv.getDictionary());
if (curShard != lastShard
|| curSliceTimestamp != lastSliceTimestamp) {
@@ -159,7 +156,7 @@ public class IIKeyValueCodec {
}
consumeCurrent(v);
}
- if (next == null) {
+ if (slice == null) {
makeNext();
}
}
@@ -176,63 +173,26 @@ public class IIKeyValueCodec {
curCol = BytesUtil.readUnsigned(buf, i, COLNO_LEN);
i += COLNO_LEN;
- if (i - k.getOffset() < k.getLength()) {
- // bitmap
- int colLen = info.length(curCol);
- curColValue = BytesUtil.readUnsigned(buf, i, colLen);
- i += colLen;
- } else {
- // value list
- curColValue = -1;
- }
}
private void consumeCurrent(ImmutableBytesWritable v) {
- if (curCol != lastCol && bitMapValues.size() > 0) { // end of a
- // bitmap
- // container
- addBitMapContainer(lastCol);
- }
- if (curColValue < 0) {
- CompressedValueContainer c = new CompressedValueContainer(info,
- curCol, 0);
- c.fromBytes(v);
- addContainer(curCol, c);
- } else {
- assert curColValue == bitMapValues.size();
- // make a copy, the value object from caller is typically reused
- // through iteration
- bitMapValues.add(new ImmutableBytesWritable(v));
- }
-
lastShard = curShard;
lastSliceTimestamp = curSliceTimestamp;
lastCol = curCol;
}
private void makeNext() {
- if (bitMapValues.isEmpty() == false) {
- addBitMapContainer(lastCol);
- }
if (containers != null) {
- next = new Slice(info, lastShard, lastSliceTimestamp,
+ slice = new Slice(info, lastShard, lastSliceTimestamp,
containers);
- next.setLocalDictionaries(Maps.newHashMap(localDictionaries));
+ slice.setLocalDictionaries(Maps.newHashMap(localDictionaries));
}
lastSliceTimestamp = Long.MIN_VALUE;
lastCol = -1;
containers = null;
- bitMapValues.clear();
localDictionaries.clear();
}
- private void addBitMapContainer(int col) {
- BitMapContainer c = new BitMapContainer(info, col);
- c.fromBytes(bitMapValues);
- addContainer(col, c);
- bitMapValues.clear();
- }
-
private void addContainer(int col, ColumnValueContainer c) {
if (containers == null) {
containers = new ColumnValueContainer[info.getColumnCount()];
@@ -246,13 +206,13 @@ public class IIKeyValueCodec {
@Override
public boolean hasNext() {
goToNext();
- return next != null;
+ return slice != null;
}
@Override
public Slice next() {
- Slice result = next;
- next = null;
+ Slice result = slice;
+ slice = null;
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
new file mode 100644
index 0000000..aba4fff
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
@@ -0,0 +1,67 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+
+/**
+ * Created by qianzhou on 3/10/15.
+ */
+public final class IIRow {
+
+ private final ImmutableBytesWritable key;
+ private final ImmutableBytesWritable value;
+ private final ImmutableBytesWritable dictionary;
+
+ public IIRow(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) {
+ this.key = key;
+ this.value = value;
+ this.dictionary = dictionary;
+ }
+ public IIRow() {
+ this(new ImmutableBytesWritable(), new ImmutableBytesWritable(), new ImmutableBytesWritable());
+ }
+
+ public ImmutableBytesWritable getKey() {
+ return key;
+ }
+
+ public ImmutableBytesWritable getValue() {
+ return value;
+ }
+ public ImmutableBytesWritable getDictionary() {
+ return dictionary;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
new file mode 100644
index 0000000..bbd8d61
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
@@ -0,0 +1,49 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.kylin.invertedindex.index.Slice;
+
+import java.util.Collection;
+
+/**
+ * Created by qianzhou on 3/12/15.
+ */
+public interface KeyValueCodec {
+
+ Collection<IIRow> encodeKeyValue(Slice slice);
+
+ Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
deleted file mode 100644
index 48c45f6..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.dict.Dictionary;
-
-/**
- * Created by qianzhou on 3/10/15.
- */
-public final class KeyValuePair {
-
- private final ImmutableBytesWritable key;
- private final ImmutableBytesWritable value;
- private final int id;
- private Dictionary<?> dictionary;
-
- public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value) {
- this(-1, key, value);
- }
- public KeyValuePair(int id, ImmutableBytesWritable key, ImmutableBytesWritable value) {
- this.id = id;
- this.key = key;
- this.value = value;
- }
-
- public void setDictionary(Dictionary<?> dictionary) {
- this.dictionary = dictionary;
- }
-
- public int getId() {
- return this.id;
- }
-
- public Dictionary<?> getDictionary() {
- return dictionary;
- }
-
- public ImmutableBytesWritable getKey() {
- return key;
- }
-
- public ImmutableBytesWritable getValue() {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
index eb65ca8..aecc37d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
@@ -33,7 +33,7 @@ import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import java.io.IOException;
import java.util.Iterator;
@@ -54,7 +54,7 @@ public class IICLI {
System.out.println("Reading from " + path + " ...");
TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment());
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+ IIKeyValueCodec codec = new IIKeyValueCodec();
int count = 0;
for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) {
for (RawTableRecord rec : slice) {
@@ -65,17 +65,17 @@ public class IICLI {
System.out.println("Total " + count + " records");
}
- public static Iterable<KeyValuePair> readSequenceKVs(
+ public static Iterable<IIRow> readSequenceKVs(
Configuration hconf, String path) throws IOException {
final Reader reader = new Reader(hconf,
SequenceFile.Reader.file(new Path(path)));
- return new Iterable<KeyValuePair>() {
+ return new Iterable<IIRow>() {
@Override
- public Iterator<KeyValuePair> iterator() {
- return new Iterator<KeyValuePair>() {
+ public Iterator<IIRow> iterator() {
+ return new Iterator<IIRow>() {
ImmutableBytesWritable k = new ImmutableBytesWritable();
ImmutableBytesWritable v = new ImmutableBytesWritable();
- KeyValuePair pair = new KeyValuePair(k, v);
+ IIRow pair = new IIRow(k, v, null);
@Override
public boolean hasNext() {
@@ -93,7 +93,7 @@ public class IICLI {
}
@Override
- public KeyValuePair next() {
+ public IIRow next() {
return pair;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index f040820..b152cdd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -29,8 +29,7 @@ import org.apache.kylin.invertedindex.index.*;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -63,33 +62,6 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
}
@Test
- public void testBitMapContainer() {
- // create container
- BitMapContainer container = new BitMapContainer(info.getDigest(), 0);
- Dictionary<String> dict = info.dict(0);
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container.append(v);
- }
- container.append(Dictionary.NULL_ID[dict.getSizeOfId()]);
- container.closeForChange();
-
- // copy by serialization
- List<ImmutableBytesWritable> bytes = container.toBytes();
- BitMapContainer container2 = new BitMapContainer(info.getDigest(), 0);
- container2.fromBytes(bytes);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- int value = container2.getValueIntAt(i++);
- assertEquals(v, value);
- }
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()],
- container2.getValueIntAt(i++));
- assertEquals(container, container2);
- }
-
- @Test
public void testCompressedValueContainer() {
// create container
CompressedValueContainer container = new CompressedValueContainer(
@@ -136,8 +108,8 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
List<Slice> slices = buildTimeSlices(records);
System.out.println(slices.size() + " slices");
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- List<KeyValuePair> kvs = encodeKVs(codec, slices);
+ IIKeyValueCodec codec = new IIKeyValueCodec();
+ List<IIRow> kvs = encodeKVs(codec, slices);
System.out.println(kvs.size() + " KV pairs");
List<Slice> slicesCopy = decodeKVs(codec, kvs);
@@ -198,10 +170,10 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
return slices;
}
- private List<KeyValuePair> encodeKVs(
+ private List<IIRow> encodeKVs(
IIKeyValueCodec codec, List<Slice> slices) {
- List<KeyValuePair> kvs = Lists
+ List<IIRow> kvs = Lists
.newArrayList();
for (Slice slice : slices) {
kvs.addAll(codec.encodeKeyValue(slice));
@@ -210,7 +182,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
}
private List<Slice> decodeKVs(IIKeyValueCodec codec,
- List<KeyValuePair> kvs) {
+ List<IIRow> kvs) {
List<Slice> slices = Lists.newArrayList();
for (Slice slice : codec.decodeKeyValue(kvs)) {
slices.add(slice);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
index 40cca36..285b058 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
@@ -31,7 +31,7 @@ import org.apache.kylin.invertedindex.index.SliceBuilder;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -60,7 +60,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
info = new TableRecordInfo(seg);
rec = info.createTableRecord();
builder = null;
- kv = new IIKeyValueCodec(info.getDigest());
+ kv = new IIKeyValueCodec();
}
@Override
@@ -92,7 +92,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
}
private void output(Slice slice, Context context) throws IOException, InterruptedException {
- for (KeyValuePair pair : kv.encodeKeyValue(slice)) {
+ for (IIRow pair : kv.encodeKeyValue(slice)) {
context.write(pair.getKey(), pair.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 29b2bcf..05d2421 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -29,15 +29,14 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
/**
* @author yangli9
*
*/
-public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable {
+public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
byte[] family;
@@ -60,15 +59,15 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
}
@Override
- public Iterator<KeyValuePair> iterator() {
+ public Iterator<IIRow> iterator() {
return new MyIterator();
}
- private class MyIterator implements Iterator<KeyValuePair> {
+ private class MyIterator implements Iterator<IIRow> {
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
- KeyValuePair pair = new KeyValuePair(key, value);
+ IIRow pair = new IIRow(key, value);
@Override
public boolean hasNext() {
@@ -76,7 +75,7 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
}
@Override
- public KeyValuePair next() {
+ public IIRow next() {
Result r = iterator.next();
Cell c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index 47aa7d9..3dbc8ea 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -18,21 +18,17 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
+import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
import java.io.Closeable;
-import java.io.DataInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -41,13 +37,11 @@ import java.util.List;
/**
* Created by honma on 11/10/14.
*/
-public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable {
+public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
private RegionScanner innerScanner;
private Logger logger = LoggerFactory.getLogger(HbaseServerKVIterator.class);
- List<Cell> results = new ArrayList<Cell>();
-
public HbaseServerKVIterator(RegionScanner innerScanner) {
this.innerScanner = innerScanner;
}
@@ -57,69 +51,55 @@ public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable
IOUtils.closeQuietly(this.innerScanner);
}
- @Override
- public Iterator<KeyValuePair> iterator() {
- return new Iterator<KeyValuePair>() {
+ private static class IIRowIterator implements Iterator<IIRow> {
- ImmutableBytesWritable key = new ImmutableBytesWritable();
- ImmutableBytesWritable value = new ImmutableBytesWritable();
- ImmutableBytesWritable dict = new ImmutableBytesWritable();
- KeyValuePair pair = new KeyValuePair(key, value);
+ private final RegionScanner regionScanner;
+ private final IIRow row = new IIRow();
+ private boolean hasMore = true;
+ List<Cell> results = Lists.newArrayList();
- private boolean hasMore = true;
+ IIRowIterator(RegionScanner innerScanner) {
+ this.regionScanner = innerScanner;
+ }
- @Override
- public boolean hasNext() {
- return hasMore;
- }
- @Override
- public KeyValuePair next() {
- if (hasNext()) {
- try {
- hasMore = innerScanner.nextRaw(results);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- if (results.size() < 1)
- throw new IllegalStateException("Hbase row contains less than 1 cell");
-
- Dictionary<?> dictionary = null;
- boolean hasDictData = false;
- for (Cell c : results) {
- if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
- key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
- dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- hasDictData = true;
- }
- }
- if (hasDictData) {
- try {
- final DataInputStream in = new DataInputStream(new ByteArrayInputStream(dict.get(), dict.getOffset(), dict.getLength()));
- String type = in.readUTF();
- dictionary = (Dictionary<?>) ClassUtil.forName(type, Dictionary.class).newInstance();
- dictionary.readFields(in);
- } catch (Exception e) {
- logger.error("error create dictionary", e);
- }
- }
- pair.setDictionary(dictionary);
-
- results.clear();
- return pair;
- } else {
- return null;
+ @Override
+ public boolean hasNext() {
+ return hasMore;
+ }
+
+ @Override
+ public IIRow next() {
+ results.clear();
+ try {
+ hasMore = regionScanner.next(results);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (results.size() < 1) {
+ throw new IllegalStateException("Hbase row contains less than 1 cell");
+ }
+ for (Cell c : results) {
+ if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
+ row.getKey().set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
+ row.getValue().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
+ row.getDictionary().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
}
}
+ return row;
+ }
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+
+ @Override
+ public Iterator<IIRow> iterator() {
+ return new IIRowIterator(innerScanner);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index fbe3e8b..a20104d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -22,19 +22,7 @@ import com.google.protobuf.ByteString;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
-
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
-import org.apache.kylin.storage.hbase.coprocessor.*;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
-
import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -45,7 +33,15 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
+import org.apache.kylin.storage.hbase.coprocessor.*;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
import java.io.IOException;
import java.util.Iterator;
@@ -94,7 +90,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
region.startRegionOperation();
synchronized (innerScanner) {
- IIKeyValueCodec codec = new IIKeyValueCodec(tableRecordInfoDigest);
+ IIKeyValueCodec codec = new IIKeyValueCodec();
//TODO pass projector to codec to skip loading columns
Iterable<Slice> slices = codec.decodeKeyValue(new HbaseServerKVIterator(innerScanner));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 5ed48cf..10d212a 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.invertedindex.index.Slice;
@@ -51,7 +50,7 @@ import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -104,7 +103,7 @@ public class IIStreamBuilder extends StreamBuilder {
return parseStream(input, desc);
}
});
- final Map<TblColRef, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
+ final Map<Integer, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap = Maps.newHashMap();
int index = 0;
for (TblColRef tblColRef : desc.listAllColumns()) {
@@ -115,14 +114,14 @@ public class IIStreamBuilder extends StreamBuilder {
}
TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap, measureCodecMap);
SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
- final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
- loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()), desc.listAllColumns(), dictionaryMap);
+ final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo, dictionaryMap);
+ loadToHBase(hTable, slice, new IIKeyValueCodec());
submitOffset();
stopwatch.stop();
logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS);
}
- private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
+ private Map<Integer, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
Set<TblColRef> dimensionColumns = Sets.newHashSet();
for (int i = 0; i < desc.listAllColumns().size(); i++) {
@@ -136,9 +135,9 @@ public class IIStreamBuilder extends StreamBuilder {
valueMap.put(desc.listAllColumns().get(i), cell);
}
}
- Map<TblColRef, Dictionary<?>> result = Maps.newHashMap();
+ Map<Integer, Dictionary<?>> result = Maps.newHashMap();
for (TblColRef tblColRef : valueMap.keys()) {
- result.put(tblColRef, DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
+ result.put(desc.findColumn(tblColRef), DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
@Nullable
@Override
public byte[] apply(String input) {
@@ -153,7 +152,7 @@ public class IIStreamBuilder extends StreamBuilder {
return Lists.newArrayList(new String(stream.getRawData()).split(","));
}
- private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo) {
+ private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo, Map<Integer, Dictionary<?>> localDictionary) {
for (List<String> row : table) {
TableRecord tableRecord = tableRecordInfo.createTableRecord();
for (int i = 0; i < row.size(); i++) {
@@ -161,31 +160,35 @@ public class IIStreamBuilder extends StreamBuilder {
}
sliceBuilder.append(tableRecord);
}
- return sliceBuilder.close();
+ final Slice slice = sliceBuilder.close();
+ slice.setLocalDictionaries(localDictionary);
+ return slice;
}
- private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec, List<TblColRef> allColumns, Map<TblColRef, Dictionary<?>> localDictionaries) throws IOException {
+ private byte[] getDictBytes(Dictionary<?> dict) throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ out.writeUTF(dict.getClass().getName());
+ dict.write(out);
+ return baos.toByteArray();
+ }
+
+ private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
try {
List<Put> data = Lists.newArrayList();
- for (KeyValuePair pair : codec.encodeKeyValue(slice)) {
- final byte[] key = pair.getKey().get();
- final byte[] value = pair.getValue().get();
+ for (IIRow row : codec.encodeKeyValue(slice)) {
+ final byte[] key = row.getKey().get();
+ final byte[] value = row.getValue().get();
Put put = new Put(key);
put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
-
- //dictionary
- final int id = pair.getId();
- if (id >= 0 && id < allColumns.size()) {
- final Dictionary<?> dictionary = localDictionaries.get(allColumns.get(id));
- if (dictionary != null) {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos);
- out.writeUTF(dictionary.getClass().getName());
- dictionary.write(out);
- put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, baos.toByteArray());
- }
- data.add(put);
+ final ImmutableBytesWritable dictionary = row.getDictionary();
+ final byte[] dictBytes = dictionary.get();
+ if (dictionary.getOffset() == 0 && dictionary.getLength() == dictBytes.length) {
+ put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
+ } else {
+ throw new RuntimeException("dict offset should be 0, and dict length should be " + dictBytes.length + " but they are" + dictionary.getOffset() + " " + dictionary.getLength());
}
+ data.add(put);
}
hTable.put(data);
} finally {
[47/50] [abbrv] incubator-kylin git commit: KYLIN-625, draft impl,
ready for very first test
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
new file mode 100644
index 0000000..a80f4d9
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -0,0 +1,169 @@
+package org.apache.kylin.storage.gridtable;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreScanner;
+
+class GTRawScanner implements IGTScanner {
+
+ final GTInfo info;
+ final IGTStoreScanner storeScanner;
+ final TupleFilter filter;
+ final BitSet selectedColBlocks;
+
+ private GTRowBlock currentBlock;
+ private int currentRow;
+ private GTRecord next;
+ final private GTRecord oneRecord; // avoid instance creation
+ final private TupleAdapter oneTuple; // avoid instance creation
+
+ GTRawScanner(GTInfo info, IGTStore store, GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ this.info = info;
+ this.filter = filter;
+
+ ByteArray start = pkStart.exportColumns(info.primaryKey);
+ ByteArray endEx = pkEndExclusive.exportColumns(info.primaryKey);
+
+ ConciseSet selectedRowBlocks = computeHitRowBlocks(start, endEx, filter);
+ this.selectedColBlocks = computeHitColumnBlocks(dimensions, metrics);
+
+ this.storeScanner = store.scan(start, endEx, selectedRowBlocks, selectedColBlocks);
+ this.oneRecord = new GTRecord(info);
+ this.oneTuple = new TupleAdapter(oneRecord);
+ }
+
+ private BitSet computeHitColumnBlocks(BitSet dimensions, BitSet metrics) {
+ BitSet result = new BitSet();
+ for (int i = 0; i < info.colBlocks.length; i++) {
+ BitSet cb = info.colBlocks[i];
+ if (cb.intersects(dimensions) || cb.intersects(metrics)) {
+ result.set(i);
+ }
+ }
+ return result;
+ }
+
+ private ConciseSet computeHitRowBlocks(ByteArray start, ByteArray endEx, TupleFilter filter) {
+ // TODO block level index
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ storeScanner.close();
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return new Iterator<GTRecord>() {
+
+ @Override
+ public boolean hasNext() {
+ if (next != null)
+ return true;
+
+ IFilterCodeSystem<ByteArray> filterCodeSystem = info.codeSystem.getFilterCodeSystem();
+
+ while (fetchNext()) {
+ if (filter != null && filter.evaluate(oneTuple, filterCodeSystem) == false) {
+ continue;
+ }
+ next = oneRecord;
+ return true;
+ }
+ return false;
+ }
+
+ private boolean fetchNext() {
+ if (info.isRowBlockEnabled()) {
+ return fetchNextRowBlockEnabled();
+ } else {
+ return fetchNextRowBlockDisabled();
+ }
+ }
+
+ private boolean fetchNextRowBlockDisabled() {
+ // row block disabled, every block is one row
+ if (storeScanner.hasNext() == false)
+ return false;
+
+ // when row block disabled, PK is persisted in block primary key (not in cell block)
+ currentBlock = storeScanner.next();
+ oneRecord.loadPrimaryKey(currentBlock.primaryKeyBuffer);
+ for (int c = selectedColBlocks.nextSetBit(0); c >= 0; c = selectedColBlocks.nextSetBit(c + 1)) {
+ oneRecord.loadCellBlock(c, currentBlock.cellBlockBuffers[c]);
+ }
+ return true;
+ }
+
+ private boolean fetchNextRowBlockEnabled() {
+ while (true) {
+ // get a block
+ if (currentBlock == null) {
+ if (storeScanner.hasNext()) {
+ currentBlock = storeScanner.next();
+ currentRow = 0;
+ } else {
+ return false;
+ }
+ }
+ // if block exhausted, try next block
+ if (currentRow >= currentBlock.nRows) {
+ currentBlock = null;
+ continue;
+ }
+ // fetch a row
+ for (int c = selectedColBlocks.nextSetBit(0); c >= 0; c = selectedColBlocks.nextSetBit(c + 1)) {
+ oneRecord.loadCellBlock(c, currentBlock.cellBlockBuffers[c]);
+ }
+ return true;
+ }
+ }
+
+ @Override
+ public GTRecord next() {
+ // fetch next record
+ if (next == null) {
+ hasNext();
+ if (next == null)
+ throw new NoSuchElementException();
+ }
+
+ GTRecord result = next;
+ next = null;
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ private static class TupleAdapter implements IEvaluatableTuple {
+
+ private GTRecord r;
+
+ private TupleAdapter(GTRecord r) {
+ this.r = r;
+ }
+
+ @Override
+ public Object getValue(TblColRef col) {
+ return r.cols[col.getColumn().getZeroBasedIndex()];
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 7943965..146d4c7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -1,15 +1,140 @@
package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
+import java.util.BitSet;
-public class GTRecord {
+import org.apache.kylin.common.util.ByteArray;
+
+public class GTRecord implements Comparable<GTRecord> {
final GTInfo info;
- final ByteBuffer[] cols;
+ final ByteArray[] cols;
+
+ BitSet maskForEqualHashComp;
public GTRecord(GTInfo info) {
this.info = info;
- this.cols = new ByteBuffer[info.nColumns];
+ this.cols = new ByteArray[info.nColumns];
+ for (int i = 0; i < cols.length; i++)
+ this.cols[i] = new ByteArray();
+ this.maskForEqualHashComp = info.colAll;
+ }
+
+ public GTRecord copy() {
+ return copy(info.colAll);
+ }
+
+ public GTRecord copy(BitSet selectedCols) {
+ int len = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ len += cols[i].length();
+ }
+
+ byte[] space = new byte[len];
+
+ GTRecord copy = new GTRecord(info);
+ copy.maskForEqualHashComp = this.maskForEqualHashComp;
+ int pos = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ System.arraycopy(cols[i].array(), cols[i].offset(), space, pos, cols[i].length());
+ copy.cols[i].set(space, pos, cols[i].length());
+ pos += cols[i].length();
+ }
+
+ return copy;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+
+ GTRecord o = (GTRecord) obj;
+ if (this.info != o.info)
+ return false;
+ if (this.maskForEqualHashComp != o.maskForEqualHashComp)
+ return false;
+ for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
+ if (this.cols[i].equals(o.cols[i]) == false)
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 1;
+ for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
+ hash = (31 * hash) + cols[i].hashCode();
+ }
+ return hash;
+ }
+
+ @Override
+ public int compareTo(GTRecord o) {
+ int comp = 0;
+ for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
+ comp = this.cols[i].compareTo(o.cols[i]);
+ if (comp != 0)
+ return comp;
+ }
+ return comp;
+ }
+
+
+ // ============================================================================
+
+ ByteArray exportColumns(BitSet selectedCols) {
+ int len = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ len += cols[i].length();
+ }
+
+ ByteArray buf = ByteArray.allocate(len);
+ exportColumns(info.primaryKey, buf);
+ return buf;
+ }
+
+ /** write data to given buffer, like serialize */
+ void exportColumns(BitSet selectedCols, ByteArray buf) {
+ int pos = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ System.arraycopy(cols[i].array(), cols[i].offset(), buf.array(), buf.offset() + pos, cols[i].length());
+ pos += cols[i].length();
+ }
+ buf.setLength(pos);
+ }
+
+ /** write data to given buffer, like serialize */
+ void exportColumnBlock(int c, ByteBuffer buf) {
+ BitSet setselectedCols = info.colBlocks[c];
+ for (int i = setselectedCols.nextSetBit(0); i >= 0; i = setselectedCols.nextSetBit(i + 1)) {
+ buf.put(cols[i].array(), cols[i].offset(), cols[i].length());
+ }
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ void loadPrimaryKey(ByteBuffer buf) {
+ loadColumns(info.primaryKey, buf);
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ void loadCellBlock(int c, ByteBuffer buf) {
+ loadColumns(info.colBlocks[c], buf);
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ void loadColumns(BitSet selectedCols, ByteBuffer buf) {
+ int pos = buf.position();
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ int len = info.codeSystem.codeLength(i, buf);
+ cols[i].set(buf.array(), buf.arrayOffset() + pos, len);
+ buf.position(pos + len);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index 6db08ab..ca4a35d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -2,33 +2,114 @@ package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
-class GTRowBlock {
+import org.apache.kylin.common.util.ByteArray;
- private static final int PRIMARY_KEY_CAPACITY = 2048;
- private static final int CELL_BLOCK_CAPACITY = 128 * 1024;
+public class GTRowBlock {
+ /** create a row block, allocate memory, get ready for incoming data */
+ public static GTRowBlock allocate(GTInfo info) {
+ GTRowBlock b = new GTRowBlock(info);
+
+ byte[] array = new byte[info.maxRecordLength];
+ b.primaryKey.set(array);
+ b.primaryKeyBuffer = ByteBuffer.wrap(array);
+
+ int maxRows = info.isRowBlockEnabled() ? info.rowBlockSize : 1;
+ for (int i = 0; i < b.cellBlocks.length; i++) {
+ array = new byte[info.maxRecordLength * maxRows];
+ b.cellBlocks[i].set(array);
+ b.cellBlockBuffers[i] = ByteBuffer.wrap(array);
+ }
+ return b;
+ }
+
+ final GTInfo info;
+
int seqId; // 0, 1, 2...
int nRows;
- ByteBuffer primaryKey; // the primary key of the first row
- ByteBuffer[] cellBlocks; // cells for each column block
+ ByteArray primaryKey; // the primary key of the first row
+ ByteBuffer primaryKeyBuffer;
+ ByteArray[] cellBlocks; // cells for each column block
+ ByteBuffer[] cellBlockBuffers;
+ /** create a row block that has no underlying memory */
public GTRowBlock(GTInfo info) {
- primaryKey = ByteBuffer.allocate(PRIMARY_KEY_CAPACITY);
- cellBlocks = new ByteBuffer[info.nColBlocks];
+ this.info = info;
+ primaryKey = new ByteArray();
+ cellBlocks = new ByteArray[info.colBlocks.length];
for (int i = 0; i < cellBlocks.length; i++) {
- cellBlocks[i] = ByteBuffer.allocate(CELL_BLOCK_CAPACITY);
+ cellBlocks[i] = new ByteArray();
}
+ cellBlockBuffers = new ByteBuffer[info.colBlocks.length];
+ }
+
+ public GTRowBlock copy() {
+ GTRowBlock copy = new GTRowBlock(info);
+
+ ByteBuffer buf = ByteBuffer.allocate(this.exportLength());
+ this.export(buf);
+ buf.clear();
+ copy.load(buf);
+
+ return copy;
}
+ public void rewindBuffers() {
+ primaryKeyBuffer.rewind();
+ for (int i = 0; i < cellBlockBuffers.length; i++) {
+ cellBlockBuffers[i].rewind();
+ }
+ }
+
public boolean isEmpty() {
return nRows == 0;
}
public void clear() {
nRows = 0;
- primaryKey.clear();
- for (int i = 0; i < cellBlocks.length; i++) {
- cellBlocks[i].clear();
+ rewindBuffers();
+ }
+
+ public int exportLength() {
+ int len = 4 + 4 + (4 + primaryKey.length());
+ for (ByteArray array : cellBlocks) {
+ len += 4 + array.length();
+ }
+ return len;
+ }
+
+ /** write data to given buffer, like serialize */
+ public void export(ByteBuffer buf) {
+ buf.putInt(seqId);
+ buf.putInt(nRows);
+ export(primaryKey, buf);
+ for (ByteArray cb : cellBlocks) {
+ export(cb, buf);
}
}
+
+ private void export(ByteArray array, ByteBuffer buf) {
+ buf.putInt(array.length());
+ buf.put(array.array(), array.offset(), array.length());
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ public void load(ByteBuffer buf) {
+ seqId = buf.getInt();
+ nRows = buf.getInt();
+ load(primaryKey, buf);
+ primaryKeyBuffer = ByteBuffer.wrap(primaryKey.array(), primaryKey.offset(), primaryKey.length());
+ for (int i = 0; i < info.colBlocks.length; i++) {
+ ByteArray cb = cellBlocks[i];
+ load(cb, buf);
+ cellBlockBuffers[i] = ByteBuffer.wrap(cb.array(), cb.offset(), cb.length());
+ }
+ }
+
+ private void load(ByteArray array, ByteBuffer buf) {
+ int len = buf.getInt();
+ int pos = buf.position();
+ array.set(buf.array(), buf.arrayOffset() + pos, len);
+ buf.position(pos + len);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
deleted file mode 100644
index b665fd6..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-public interface GTStore {
-
- public GTInfo getInfo();
-
- public String getStorageDescription();
-
- // ============================================================================
-
- public GTWriter rebuild(int shard);
-
- public GTScanner scan(ByteBuffer pkStart, ByteBuffer pkEndExclusive, ConciseSet selectedRowBlcoks, int[] selectedColBlocks);
-
- public interface GTWriter extends Closeable {
- void write(GTRowBlock block) throws IOException;
- }
-
- public interface GTScanner extends Iterator<GTRowBlock>, Closeable {
- }
-
- // ============================================================================
-
- public void saveRowBlockIndex(int col, GTRowBlockIndex index);
-
- public GTRowBlockIndex loadRowBlockIndex(int col);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
index 7eb8657..5d09928 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
@@ -1,19 +1,34 @@
package org.apache.kylin.storage.gridtable;
+import java.util.BitSet;
+
+import org.apache.kylin.metadata.filter.TupleFilter;
+
public class GridTable {
final GTInfo info;
+ final IGTStore store;
- public GridTable(GTInfo info, GTStore store) {
+ public GridTable(GTInfo info, IGTStore store) {
this.info = info;
+ this.store = store;
}
- public GTBuilder rebuild(GTStore store) {
- assert info.nShards == 0;
- return rebuild(store, 0);
+ public GTBuilder rebuild() {
+ assert info.isShardingEnabled() == false;
+ return rebuild(-1);
}
- public GTBuilder rebuild(GTStore store, int shard) {
+ public GTBuilder rebuild(int shard) {
+ assert shard < info.nShards;
return new GTBuilder(info, shard, store);
}
+
+ public IGTScanner scan(GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ return new GTRawScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics, filter);
+ }
+
+ public IGTScanner scanAndAggregate(GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ return new GTAggregateScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics, filter);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
new file mode 100644
index 0000000..2e960a0
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
@@ -0,0 +1,25 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+
+public interface IGTCodeSystem {
+
+ IFilterCodeSystem<ByteArray> getFilterCodeSystem();
+
+ /** return the length of code starting at the specified buffer, buffer position must not change after return */
+ int codeLength(int col, ByteBuffer buf);
+
+ /** encode a value into code */
+ void encodeColumnValue(int col, Object value, ByteBuffer buf);
+
+ /** decode a code into value */
+ Object decodeColumnValue(int col, ByteBuffer buf);
+
+ /** return an aggregator for metrics */
+ MeasureAggregator<?> newMetricsAggregator(int col);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
new file mode 100644
index 0000000..c51c301
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
@@ -0,0 +1,7 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+
+public interface IGTScanner extends Iterable<GTRecord>, Closeable {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
new file mode 100644
index 0000000..621b66b
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
@@ -0,0 +1,37 @@
+package org.apache.kylin.storage.gridtable;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.Iterator;
+
+import org.apache.kylin.common.util.ByteArray;
+
+public interface IGTStore {
+
+ public GTInfo getInfo();
+
+ public String getStorageDescription();
+
+ // ============================================================================
+
+ public IGTStoreWriter rebuild(int shard);
+
+ public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEndExclusive, ConciseSet selectedRowBlocks, BitSet selectedColBlocks);
+
+ public interface IGTStoreWriter extends Closeable {
+ void write(GTRowBlock block) throws IOException;
+ }
+
+ public interface IGTStoreScanner extends Iterator<GTRowBlock>, Closeable {
+ }
+
+ // ============================================================================
+
+ public void saveRowBlockIndex(int col, GTRowBlockIndex index);
+
+ public GTRowBlockIndex loadRowBlockIndex(int col);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
deleted file mode 100644
index d62d478..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.util.Pair;
-
-public interface IKVStoreReader extends Iterator<Pair<ByteBuffer, ByteBuffer>>, Closeable {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
deleted file mode 100644
index 8eeba95..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public interface IKVStoreWriter extends Closeable {
-
- void write(ByteBuffer key, ByteBuffer value) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
new file mode 100644
index 0000000..f342d7b
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
@@ -0,0 +1,96 @@
+package org.apache.kylin.storage.gridtable.memstore;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRowBlock;
+import org.apache.kylin.storage.gridtable.GTRowBlockIndex;
+import org.apache.kylin.storage.gridtable.IGTStore;
+
+public class GTSimpleMemStore implements IGTStore {
+
+ final GTInfo info;
+ final List<GTRowBlock> rowBlockList;
+
+ public GTSimpleMemStore(GTInfo info) {
+ this.info = info;
+ this.rowBlockList = new ArrayList<GTRowBlock>();
+
+ if (info.isShardingEnabled())
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public GTInfo getInfo() {
+ return info;
+ }
+
+ @Override
+ public String getStorageDescription() {
+ return this.toString();
+ }
+
+ @Override
+ public IGTStoreWriter rebuild(int shard) {
+
+ rowBlockList.clear();
+
+ return new IGTStoreWriter() {
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void write(GTRowBlock block) throws IOException {
+ rowBlockList.add(block.copy());
+ }
+ };
+ }
+
+ @Override
+ public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEndExclusive, ConciseSet selectedRowBlocks, BitSet selectedColBlocks) {
+
+ return new IGTStoreScanner() {
+ Iterator<GTRowBlock> it = rowBlockList.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return it.hasNext();
+ }
+
+ @Override
+ public GTRowBlock next() {
+ GTRowBlock block = it.next();
+ block.rewindBuffers();
+ return block;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ }
+
+ @Override
+ public void saveRowBlockIndex(int col, GTRowBlockIndex index) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public GTRowBlockIndex loadRowBlockIndex(int col) {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
index cada02b..bbe7c36 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
@@ -4,9 +4,14 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
-public class DictCodeSystem implements ICodeSystem<String> {
+/**
+ * A simple code system where all values are dictionary IDs (fixed length bytes) encoded as ISO-8859-1 strings.
+ *
+ * @author yangli9
+ */
+public class DictCodeSystem implements IFilterCodeSystem<String> {
public static final DictCodeSystem INSTANCE = new DictCodeSystem();
@@ -15,16 +20,6 @@ public class DictCodeSystem implements ICodeSystem<String> {
}
@Override
- public String encode(Object value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object decode(String code) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public boolean isNull(String value) {
if (value == null)
return true;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
index fe9d336..95c6555 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -38,7 +37,7 @@ import org.apache.kylin.storage.tuple.TupleInfo;
*/
public class FilterBaseTest {
- static final ICodeSystem CS = StringCodeSystem.INSTANCE;
+ static final IFilterCodeSystem CS = StringCodeSystem.INSTANCE;
protected List<TblColRef> buildGroups() {
List<TblColRef> groups = new ArrayList<TblColRef>();
[46/50] [abbrv] incubator-kylin git commit: fix
Posted by li...@apache.org.
fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/50597236
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/50597236
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/50597236
Branch: refs/heads/streaming-localdict
Commit: 505972360f140b9f4ff9936eabf279c3363fff47
Parents: afd35a0
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 18:18:10 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 18:18:10 2015 +0800
----------------------------------------------------------------------
job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/50597236/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index af3c8e4..c916bd8 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -94,6 +94,7 @@ public class BuildIIWithStreamTest {
@Before
public void before() throws Exception {
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
kylinConfig = KylinConfig.getInstanceFromEnv();
iiManager = IIManager.getInstance(kylinConfig);
[24/50] [abbrv] incubator-kylin git commit: fix compile issue
Posted by li...@apache.org.
fix compile issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0939359d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0939359d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0939359d
Branch: refs/heads/streaming-localdict
Commit: 0939359dcdfd384112d504d7838e32bbd295a663
Parents: e07c702
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 17:42:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 17:42:16 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/HBaseClientKVIterator.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0939359d/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 7bbe363..339612d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -31,12 +31,13 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.KeyValuePair;
/**
* @author yangli9
*
*/
-public class HBaseClientKVIterator implements Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>, Closeable {
+public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable {
byte[] family;
byte[] qualifier;
@@ -61,15 +62,15 @@ public class HBaseClientKVIterator implements Iterable<Pair<ImmutableBytesWritab
}
@Override
- public Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> iterator() {
+ public Iterator<KeyValuePair> iterator() {
return new MyIterator();
}
- private class MyIterator implements Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> {
+ private class MyIterator implements Iterator<KeyValuePair> {
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
- Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair = new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(key, value);
+ KeyValuePair pair = new KeyValuePair(key, value);
@Override
public boolean hasNext() {
@@ -77,7 +78,7 @@ public class HBaseClientKVIterator implements Iterable<Pair<ImmutableBytesWritab
}
@Override
- public Pair<ImmutableBytesWritable, ImmutableBytesWritable> next() {
+ public KeyValuePair next() {
Result r = iterator.next();
Cell c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
[14/50] [abbrv] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/59a1d665
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/59a1d665
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/59a1d665
Branch: refs/heads/streaming-localdict
Commit: 59a1d6658301cdd6d4cbcd52daafdb1480389d3a
Parents: 76e5fd3 5f9e2c9
Author: lukehan <lu...@apache.org>
Authored: Tue Mar 10 09:47:08 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Tue Mar 10 09:47:08 2015 -0700
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
14 files changed, 115 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
[21/50] [abbrv] incubator-kylin git commit: stream building
Posted by li...@apache.org.
stream building
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/66902d01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/66902d01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/66902d01
Branch: refs/heads/streaming-localdict
Commit: 66902d017785ef5395405850fbd858ca612b722d
Parents: 2be4e32
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 11 15:40:59 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 11 15:40:59 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/invertedindex/index/Slice.java | 8 +++--
.../invertedindex/index/TableRecordInfo.java | 17 ---------
.../invertedindex/model/IIKeyValueCodec.java | 15 ++++----
.../kylin/invertedindex/model/KeyValuePair.java | 23 +++++++++----
.../endpoint/HbaseServerKVIterator.java | 36 +++++++++++++++++---
.../hbase/coprocessor/endpoint/IIEndpoint.java | 3 +-
.../apache/kylin/streaming/StreamBuilder.java | 28 ++++++++-------
.../invertedindex/IIStreamBuilder.java | 31 ++++++++++-------
8 files changed, 100 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index 4ac3f01..39a04cf 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -42,6 +42,7 @@ public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
long timestamp;
int nRecords;
ColumnValueContainer[] containers;
+ private Map<Integer, Dictionary<?>> localDictionaries;
public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) {
this.info = digest;
@@ -59,8 +60,11 @@ public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
}
public Map<Integer, Dictionary<?>> getLocalDictionaries() {
- // TODO localdict
- return Collections.emptyMap();
+ return localDictionaries;
+ }
+
+ public void setLocalDictionaries(Map<Integer, Dictionary<?>> localDictionaries) {
+ this.localDictionaries = localDictionaries;
}
public int getRecordCount() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 68feb03..5535fcc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -77,23 +77,6 @@ public class TableRecordInfo {
this.digest = createDigest(dictionaryMap, measureCodecMap);
}
- private List<Integer> getLocalDictColumnList() {
- //TODO localdict
- return Collections.emptyList();
- }
-
-// public void updateDictionary(List<Dictionary<?>> dicts) {
-// List<Integer> columns = getLocalDictColumnList();
-//
-// if (columns.size() != dicts.size()) {
-// throw new RuntimeException("columns size not equal to dicts size");
-// }
-//
-// for (Integer index : columns) {
-// this.dictionaries[index] = dicts.get(index);
-// }
-// }
-
public TableRecordInfoDigest getDigest() {
return digest;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 81f4d56..8ea4e70 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -18,17 +18,16 @@
package org.apache.kylin.invertedindex.model;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Pair;
import com.google.common.collect.Lists;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.index.*;
/**
@@ -69,7 +68,7 @@ public class IIKeyValueCodec {
ImmutableBytesWritable key = encodeKey(slice.getShard(),
slice.getTimestamp(), col, -1);
ImmutableBytesWritable value = container.toBytes();
- return new KeyValuePair(key, value);
+ return new KeyValuePair(col, key, value);
}
private List<KeyValuePair> collectKeyValues(Slice slice,
@@ -80,7 +79,7 @@ public class IIKeyValueCodec {
for (int v = 0; v < values.size(); v++) {
ImmutableBytesWritable key = encodeKey(slice.getShard(),
slice.getTimestamp(), col, v);
- list.add(new KeyValuePair(key, values.get(v)));
+ list.add(new KeyValuePair(col, key, values.get(v)));
}
return list;
}
@@ -133,6 +132,7 @@ public class IIKeyValueCodec {
int lastCol = -1;
ColumnValueContainer[] containers = null;
List<ImmutableBytesWritable> bitMapValues = Lists.newArrayList();
+ Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
Decoder(TableRecordInfoDigest info,
Iterable<KeyValuePair> kvs) {
@@ -151,6 +151,7 @@ public class IIKeyValueCodec {
ImmutableBytesWritable k = kv.getKey();
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
+ localDictionaries.put(curCol, kv.getDictionary());
if (curShard != lastShard
|| curSliceTimestamp != lastSliceTimestamp) {
@@ -216,11 +217,13 @@ public class IIKeyValueCodec {
if (containers != null) {
next = new Slice(info, lastShard, lastSliceTimestamp,
containers);
+ next.setLocalDictionaries(Maps.newHashMap(localDictionaries));
}
lastSliceTimestamp = Long.MIN_VALUE;
lastCol = -1;
containers = null;
bitMapValues.clear();
+ localDictionaries.clear();
}
private void addBitMapContainer(int col) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
index 7e43b2a..48c45f6 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
@@ -35,27 +35,36 @@
package org.apache.kylin.invertedindex.model;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.dict.Dictionary;
/**
* Created by qianzhou on 3/10/15.
*/
public final class KeyValuePair {
- private ImmutableBytesWritable key;
- private ImmutableBytesWritable value;
- private ImmutableBytesWritable dictionary;
+ private final ImmutableBytesWritable key;
+ private final ImmutableBytesWritable value;
+ private final int id;
+ private Dictionary<?> dictionary;
public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value) {
- this(key, value, null);
+ this(-1, key, value);
}
-
- public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) {
+ public KeyValuePair(int id, ImmutableBytesWritable key, ImmutableBytesWritable value) {
+ this.id = id;
this.key = key;
this.value = value;
+ }
+
+ public void setDictionary(Dictionary<?> dictionary) {
this.dictionary = dictionary;
}
- public ImmutableBytesWritable getDictionary() {
+ public int getId() {
+ return this.id;
+ }
+
+ public Dictionary<?> getDictionary() {
return dictionary;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index 6bcae0d..47aa7d9 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -22,10 +22,17 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.ByteArrayInputStream;
import java.io.Closeable;
+import java.io.DataInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -37,6 +44,7 @@ import java.util.List;
public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable {
private RegionScanner innerScanner;
+ private Logger logger = LoggerFactory.getLogger(HbaseServerKVIterator.class);
List<Cell> results = new ArrayList<Cell>();
@@ -55,6 +63,7 @@ public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
+ ImmutableBytesWritable dict = new ImmutableBytesWritable();
KeyValuePair pair = new KeyValuePair(key, value);
private boolean hasMore = true;
@@ -76,9 +85,28 @@ public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable
if (results.size() < 1)
throw new IllegalStateException("Hbase row contains less than 1 cell");
- Cell c = results.get(0);
- key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ Dictionary<?> dictionary = null;
+ boolean hasDictData = false;
+ for (Cell c : results) {
+ if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
+ key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
+ value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
+ dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ hasDictData = true;
+ }
+ }
+ if (hasDictData) {
+ try {
+ final DataInputStream in = new DataInputStream(new ByteArrayInputStream(dict.get(), dict.getOffset(), dict.getLength()));
+ String type = in.readUTF();
+ dictionary = (Dictionary<?>) ClassUtil.forName(type, Dictionary.class).newInstance();
+ dictionary.readFields(in);
+ } catch (Exception e) {
+ logger.error("error create dictionary", e);
+ }
+ }
+ pair.setDictionary(dictionary);
results.clear();
return pair;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index b72a376..fbe3e8b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -63,7 +63,8 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
private Scan buildScan() {
Scan scan = new Scan();
- scan.addColumn(Bytes.toBytes(IIDesc.HBASE_FAMILY), Bytes.toBytes(IIDesc.HBASE_QUALIFIER));
+ scan.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
+ scan.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES);
return scan;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
index a86a9b1..39c3966 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
@@ -60,7 +60,7 @@ public abstract class StreamBuilder implements Runnable {
this.sliceSize = sliceSize;
}
- protected abstract boolean build(List<Stream> streamsToBuild);
+ protected abstract void build(List<Stream> streamsToBuild) throws Exception;
private void clearCounter() {
lastBuildTime = System.currentTimeMillis();
@@ -72,7 +72,13 @@ public abstract class StreamBuilder implements Runnable {
List<Stream> streamToBuild = Lists.newArrayList();
clearCounter();
while (true) {
- final Stream stream = streamQueue.poll(50, TimeUnit.MILLISECONDS);
+ Stream stream;
+ try {
+ stream = streamQueue.poll(50, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ logger.warn("stream queue interrupted", e);
+ continue;
+ }
if (stream == null) {
continue;
} else {
@@ -86,21 +92,19 @@ public abstract class StreamBuilder implements Runnable {
}
streamToBuild.add(stream);
if (streamToBuild.size() >= this.sliceSize) {
- if (build(streamToBuild)) {
- clearCounter();
- streamToBuild.clear();
- }
+ build(streamToBuild);
+ clearCounter();
+ streamToBuild.clear();
} else if ((System.currentTimeMillis() - lastBuildTime) > BATCH_BUILD_INTERVAL_THRESHOLD) {
- if (build(streamToBuild)) {
- clearCounter();
- streamToBuild.clear();
- }
+ build(streamToBuild);
+ clearCounter();
+ streamToBuild.clear();
} else {
continue;
}
}
- } catch (InterruptedException e) {
- logger.error("StreamBuilder has been interrupted", e);
+ } catch (Exception e) {
+ logger.error("build stream error, stop building", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/66902d01/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 36536bb..a4c7167 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -60,6 +60,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -90,7 +92,7 @@ public class IIStreamBuilder extends StreamBuilder {
}
@Override
- protected boolean build(List<Stream> streamsToBuild) {
+ protected void build(List<Stream> streamsToBuild) throws IOException {
List<List<String>> table = Lists.transform(streamsToBuild, new Function<Stream, List<String>>() {
@Nullable
@Override
@@ -110,14 +112,8 @@ public class IIStreamBuilder extends StreamBuilder {
TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap, measureCodecMap);
SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
-// try {
-// loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()));
-// submitOffset();
-// } catch (IOException e) {
-// logger.error("error load to hbase, build failed", e);
-// return false;
-// }
- return true;
+ loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()), desc.listAllColumns(), dictionaryMap);
+ submitOffset();
}
private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
@@ -162,7 +158,7 @@ public class IIStreamBuilder extends StreamBuilder {
return sliceBuilder.close();
}
- private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
+ private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec, List<TblColRef> allColumns, Map<TblColRef, Dictionary<?>> localDictionaries) throws IOException {
try {
List<Put> data = Lists.newArrayList();
for (KeyValuePair pair : codec.encodeKeyValue(slice)) {
@@ -170,9 +166,20 @@ public class IIStreamBuilder extends StreamBuilder {
final byte[] value = pair.getValue().get();
Put put = new Put(key);
put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
+
//dictionary
-// put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
- data.add(put);
+ final int id = pair.getId();
+ if (id >= 0 && id < allColumns.size()) {
+ final Dictionary<?> dictionary = localDictionaries.get(allColumns.get(id));
+ if (dictionary != null) {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ out.writeUTF(dictionary.getClass().getName());
+ dictionary.write(out);
+ put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, baos.toByteArray());
+ }
+ data.add(put);
+ }
}
hTable.put(data);
} finally {
[29/50] [abbrv] incubator-kylin git commit: Merge branch 'master' of
https://github.com/KylinOLAP/Kylin
Posted by li...@apache.org.
Merge branch 'master' of https://github.com/KylinOLAP/Kylin
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2efaf25d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2efaf25d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2efaf25d
Branch: refs/heads/streaming-localdict
Commit: 2efaf25dfab25e3e22e33383223241702a3f5300
Parents: 54dc31f e66e92a
Author: lukehan <lu...@apache.org>
Authored: Wed Mar 11 17:41:08 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Wed Mar 11 17:41:08 2015 -0700
----------------------------------------------------------------------
bin/kylin.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[08/50] [abbrv] incubator-kylin git commit: Merge pull request #447
from janzhongi/staging
Posted by li...@apache.org.
Merge pull request #447 from janzhongi/staging
config right authorize url Kylin-570
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5f9e2c9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5f9e2c9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5f9e2c9e
Branch: refs/heads/streaming-localdict
Commit: 5f9e2c9eb49a00331297658bfb50f02dfeb4fb46
Parents: 8cca80d 2e98153
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 17:18:54 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 17:18:54 2015 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 --
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/listeners.js | 7 -------
4 files changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[42/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/73c53b4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/73c53b4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/73c53b4f
Branch: refs/heads/streaming-localdict
Commit: 73c53b4f7f1b1a0cf547673b7830a8f53f655e72
Parents: ea0a623
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 15:59:22 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 15:59:22 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java | 2 +-
.../org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73c53b4f/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 6c1085e..3702609 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -44,7 +44,7 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
ResultScanner scanner;
Iterator<Result> iterator;
- public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family, byte[] qualifier) throws IOException {
+ public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family) throws IOException {
this.family = family;
this.table = hconn.getTable(tableName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73c53b4f/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index 548224c..9e37447 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -84,7 +84,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
List<Slice> slices = Lists.newArrayList();
- HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
+ HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES);
try {
for (Slice slice : codec.decodeKeyValue(kvIterator)) {
slices.add(slice);
[44/50] [abbrv] incubator-kylin git commit: fix ci
Posted by li...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/08d8e32c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/08d8e32c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/08d8e32c
Branch: refs/heads/streaming-localdict
Commit: 08d8e32c798115cf1dc8c6e98568b4d600ebce54
Parents: 5880b79
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 17:13:58 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 17:13:58 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/invertedindex/index/Slice.java | 5 +---
.../index/TableRecordInfoDigest.java | 29 ++++++++++++++++++--
2 files changed, 28 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/08d8e32c/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index 9e494f2..1fa6150 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -197,10 +197,7 @@ public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
if (timestamp != other.timestamp) {
return false;
}
- if (!Objects.equal(info, other.info)) {
- return false;
- }
- return true;
+ return Objects.equal(info, other.info);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/08d8e32c/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index cf519d7..cebdbce 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -18,13 +18,15 @@
package org.apache.kylin.invertedindex.index;
+import com.google.common.base.Objects;
+import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
-import org.apache.hadoop.io.LongWritable;
import java.nio.ByteBuffer;
+import java.util.Arrays;
/**
* Created by honma on 11/10/14.
@@ -105,7 +107,30 @@ public class TableRecordInfoDigest {
return new RawTableRecord(this);
}
- // metrics go with fixed-len codec
+ @Override
+ public int hashCode() {
+ return com.google.common.base.Objects.hashCode(nColumns, offsets, dictMaxIds, lengths, isMetric, metricDataTypes);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof TableRecordInfoDigest) {
+ TableRecordInfoDigest other = (TableRecordInfoDigest) obj;
+ return Objects.equal(this.nColumns, other.nColumns) &&
+ Arrays.equals(this.offsets, other.offsets) &&
+ Arrays.equals(this.dictMaxIds, other.dictMaxIds) &&
+ Arrays.equals(this.lengths, other.lengths) &&
+ Arrays.equals(this.isMetric, other.isMetric) &&
+ Arrays.equals(this.metricDataTypes, other.metricDataTypes);
+ } else {
+ return false;
+ }
+ }
+
+ // metrics go with fixed-len codec
@SuppressWarnings("unchecked")
public FixedLenMeasureCodec<LongWritable> codec(int col) {
// yes, all metrics are long currently
[37/50] [abbrv] incubator-kylin git commit: fix
Posted by li...@apache.org.
fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5781a0a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5781a0a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5781a0a2
Branch: refs/heads/streaming-localdict
Commit: 5781a0a2f42ef3b78e0a29d981670effcdcb438c
Parents: 5b5cb31
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 13:02:53 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 13:02:53 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/model/IIKeyValueCodec.java | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5781a0a2/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index d7372bc..2f69421 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -148,14 +148,9 @@ public class IIKeyValueCodec implements KeyValueCodec {
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
final Dictionary<?> dictionary = deserialize(kv.getDictionary());
- addContainer(curCol, new CompressedValueContainer(dictionary.getSizeOfId(), (dictionary.getMaxId() - dictionary.getMinId() + 1), (dictionary.getMaxId() - dictionary.getMinId() + 1)));
- byte[] bytes = new byte[dictionary.getSizeOfValue()];
- ImmutableBytesWritable buffer = new ImmutableBytesWritable(bytes);
- for (int i = dictionary.getMinId(); i <= dictionary.getMaxId(); ++i) {
- final int length = dictionary.getValueBytesFromId(i, bytes, 0);
- buffer.set(bytes, 0, length);
- containers[curCol].append(buffer);
- }
+ final CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), (dictionary.getMaxId() - dictionary.getMinId() + 1), 0);
+ c.fromBytes(kv.getValue());
+ addContainer(curCol, c);
localDictionaries.put(curCol, dictionary);
if (localDictionaries.size() < digest.getColumnCount()) {
continue;
@@ -236,4 +231,6 @@ public class IIKeyValueCodec implements KeyValueCodec {
}
+
+
}
[39/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/54b40664
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/54b40664
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/54b40664
Branch: refs/heads/streaming-localdict
Commit: 54b4066415a1dd48d2ceba25911d3b4754f7999b
Parents: f6e7323
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 13:47:43 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 13:47:43 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/invertedindex/index/Slice.java | 32 +++++----
.../invertedindex/model/IIKeyValueCodec.java | 75 +++++++++++++++++++-
.../invertedindex/InvertedIndexLocalTest.java | 5 +-
3 files changed, 95 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/54b40664/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index 39a04cf..9e494f2 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Objects;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.dict.Dictionary;
@@ -35,12 +36,12 @@ import org.apache.kylin.dict.Dictionary;
*/
public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
- TableRecordInfoDigest info;
- int nColumns;
+ final TableRecordInfoDigest info;
+ final int nColumns;
- short shard;
- long timestamp;
- int nRecords;
+ final short shard;
+ final long timestamp;
+ final int nRecords;
ColumnValueContainer[] containers;
private Map<Integer, Dictionary<?>> localDictionaries;
@@ -180,22 +181,25 @@ public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
*/
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
Slice other = (Slice) obj;
- if (info == null) {
- if (other.info != null)
- return false;
- } else if (!info.equals(other.info))
+ if (shard != other.shard) {
return false;
- if (shard != other.shard)
+ }
+ if (timestamp != other.timestamp) {
return false;
- if (timestamp != other.timestamp)
+ }
+ if (!Objects.equal(info, other.info)) {
return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/54b40664/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 2f69421..ad1b25c 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -18,6 +18,7 @@
package org.apache.kylin.invertedindex.model;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -113,9 +114,79 @@ public class IIKeyValueCodec implements KeyValueCodec {
@Override
public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
- return new Decoder(kvs, digest);
+ return new IIRowDecoder(digest, kvs.iterator());
+// return new Decoder(kvs, digest);
}
+ private static class IIRowDecoder implements Iterable<Slice> {
+
+ private final TableRecordInfoDigest digest;
+ private final Iterator<IIRow> iterator;
+
+ private IIRowDecoder(TableRecordInfoDigest digest, Iterator<IIRow> iterator) {
+ this.digest = digest;
+ this.iterator = iterator;
+ }
+
+ @Override
+ public Iterator<Slice> iterator() {
+ return new Iterator<Slice>() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Slice next() {
+ int columns = 0;
+ ColumnValueContainer[] valueContainers = new ColumnValueContainer[digest.getColumnCount()];
+ Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
+ boolean firstTime = true;
+ short curShard = 0;
+ long curTimestamp = 0;
+ short lastShard = 0;
+ long lastTimestamp = 0;
+ while (iterator().hasNext() && columns < digest.getColumnCount()) {
+ final IIRow row = iterator.next();
+ final ImmutableBytesWritable key = row.getKey();
+ int i = key.getOffset();
+ curShard = (short) BytesUtil.readUnsigned(key.get(), i, SHARD_LEN);
+ i += SHARD_LEN;
+ curTimestamp = BytesUtil.readLong(key.get(), i, TIMEPART_LEN);
+ i += TIMEPART_LEN;
+
+ if (!firstTime) {
+ Preconditions.checkArgument(curShard == lastShard, "shard should be equals in one slice, curShard is" + curShard + " lastShard is " + lastShard);
+ Preconditions.checkArgument(curTimestamp == lastTimestamp, "timestamp should be equals in one slice, curTimestamp is" + curTimestamp + " lastTimestamp is " + lastTimestamp);
+ }
+
+ int curCol = BytesUtil.readUnsigned(key.get(), i, COLNO_LEN);
+ CompressedValueContainer c = new CompressedValueContainer(digest, curCol, 0);
+ c.fromBytes(row.getValue());
+ valueContainers[curCol] = c;
+ localDictionaries.put(curCol, deserialize(row.getDictionary()));
+ columns++;
+ lastShard = curShard;
+ lastTimestamp = curTimestamp;
+ firstTime = false;
+ }
+ Preconditions.checkArgument(columns == digest.getColumnCount(), "column count is " + columns + " should be equals to digest.getColumnCount() " + digest.getColumnCount());
+ Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers);
+ slice.setLocalDictionaries(localDictionaries);
+ return slice;
+ }
+
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ }
+
+
private static class Decoder implements Iterable<Slice> {
private final TableRecordInfoDigest digest;
@@ -148,7 +219,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
final Dictionary<?> dictionary = deserialize(kv.getDictionary());
- final CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), (dictionary.getMaxId() - dictionary.getMinId() + 1), 0);
+ final CompressedValueContainer c = new CompressedValueContainer(digest, curCol, 0);
c.fromBytes(kv.getValue());
addContainer(curCol, c);
localDictionaries.put(curCol, dictionary);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/54b40664/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index c49a8dd..86c736d 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -131,7 +131,10 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
System.out.println(kvs.size() + " KV pairs");
List<Slice> slicesCopy = decodeKVs(codec, kvs);
- assertEquals(slices, slicesCopy);
+ assertEquals(slices.size(), slicesCopy.size());
+ for (int i = 0; i < slices.size(); i++) {
+ assertEquals(slices.get(i), slicesCopy.get(i));
+ }
List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
assertEquals(new HashSet<TableRecord>(records),
[28/50] [abbrv] incubator-kylin git commit: minor change to kafka test
Posted by li...@apache.org.
minor change to kafka test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0b13251b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0b13251b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0b13251b
Branch: refs/heads/streaming-localdict
Commit: 0b13251b57b426bbea54f6037f9a09a0778951d0
Parents: 0b71528
Author: honma <ho...@ebay.com>
Authored: Wed Mar 11 00:21:59 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Wed Mar 11 15:30:52 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/streaming/KafkaConsumer.java | 4 +++-
.../test/java/org/apache/kylin/streaming/KafkaConsumerTest.java | 2 ++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0b13251b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
index a9f5679..60f85d8 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
@@ -112,10 +112,12 @@ public class KafkaConsumer implements Runnable {
}
for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topic, partitionId)) {
final ByteBuffer payload = messageAndOffset.message().payload();
+
//TODO use ByteBuffer maybe
byte[] bytes = new byte[payload.limit()];
payload.get(bytes);
- logger.debug("get message offset:" + messageAndOffset.offset());
+ logger.info("get message offset:" + messageAndOffset.offset());
+ logger.info(new String(bytes));
try {
streamQueue.put(new Stream(messageAndOffset.offset(), bytes));
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0b13251b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
index 91e06fc..a6176ab 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
@@ -92,6 +92,8 @@ public class KafkaConsumerTest extends KafkaBaseTest {
for (BlockingQueue<Stream> queue : queues) {
count += queue.size();
}
+
+ logger.info("count of messages are " + count);
//since there will be historical data
assertTrue(count >= TOTAL_SEND_COUNT);
}
[12/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/af6372de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/af6372de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/af6372de
Branch: refs/heads/streaming-localdict
Commit: af6372deabad2f56e281c775184e9b209832361d
Parents: 1142b68
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Mar 10 18:25:33 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Mar 10 18:25:33 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/invertedindex/index/Slice.java | 6 ++++--
.../invertedindex/model/IIKeyValueCodec.java | 17 ++++++++--------
.../kylin/invertedindex/model/KeyValuePair.java | 14 ++++++++-----
.../apache/kylin/invertedindex/tools/IICLI.java | 21 ++++++++++----------
.../invertedindex/InvertedIndexLocalTest.java | 10 +++++-----
.../endpoint/HbaseServerKVIterator.java | 21 ++++++++++----------
.../coprocessor/endpoint/LocalDictionary.java | 10 +++++-----
.../invertedindex/IIStreamBuilder.java | 7 ++++---
8 files changed, 56 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index 56e25f0..4ac3f01 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -18,8 +18,10 @@
package org.apache.kylin.invertedindex.index;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -56,9 +58,9 @@ public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
}
}
- public List<Dictionary<?>> getLocalDictionaries() {
+ public Map<Integer, Dictionary<?>> getLocalDictionaries() {
// TODO localdict
- return null;
+ return Collections.emptyMap();
}
public int getRecordCount() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 070b672..81f4d56 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -69,7 +69,7 @@ public class IIKeyValueCodec {
ImmutableBytesWritable key = encodeKey(slice.getShard(),
slice.getTimestamp(), col, -1);
ImmutableBytesWritable value = container.toBytes();
- return new KeyValuePair(col, key, value);
+ return new KeyValuePair(key, value);
}
private List<KeyValuePair> collectKeyValues(Slice slice,
@@ -80,7 +80,7 @@ public class IIKeyValueCodec {
for (int v = 0; v < values.size(); v++) {
ImmutableBytesWritable key = encodeKey(slice.getShard(),
slice.getTimestamp(), col, v);
- list.add(new KeyValuePair(col, key, values.get(v)));
+ list.add(new KeyValuePair(key, values.get(v)));
}
return list;
}
@@ -114,14 +114,14 @@ public class IIKeyValueCodec {
}
public Iterable<Slice> decodeKeyValue(
- Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
+ Iterable<KeyValuePair> kvs) {
return new Decoder(infoDigest, kvs);
}
private static class Decoder implements Iterable<Slice> {
TableRecordInfoDigest info;
- Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> iterator;
+ Iterator<KeyValuePair> iterator;
Slice next = null;
short curShard = Short.MIN_VALUE;
@@ -135,7 +135,7 @@ public class IIKeyValueCodec {
List<ImmutableBytesWritable> bitMapValues = Lists.newArrayList();
Decoder(TableRecordInfoDigest info,
- Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
+ Iterable<KeyValuePair> kvs) {
this.info = info;
this.iterator = kvs.iterator();
}
@@ -147,10 +147,9 @@ public class IIKeyValueCodec {
// NOTE the input keys are ordered
while (next == null && iterator.hasNext()) {
- Pair<ImmutableBytesWritable, ImmutableBytesWritable> kv = iterator
- .next();
- ImmutableBytesWritable k = kv.getFirst();
- ImmutableBytesWritable v = kv.getSecond();
+ KeyValuePair kv = iterator.next();
+ ImmutableBytesWritable k = kv.getKey();
+ ImmutableBytesWritable v = kv.getValue();
decodeKey(k);
if (curShard != lastShard
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
index 445389f..7e43b2a 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
@@ -41,18 +41,22 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
*/
public final class KeyValuePair {
- private int columnIndex;
private ImmutableBytesWritable key;
private ImmutableBytesWritable value;
+ private ImmutableBytesWritable dictionary;
- public KeyValuePair(int columnIndex, ImmutableBytesWritable key, ImmutableBytesWritable value) {
- this.columnIndex = columnIndex;
+ public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value) {
+ this(key, value, null);
+ }
+
+ public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) {
this.key = key;
this.value = value;
+ this.dictionary = dictionary;
}
- public int getColumnIndex() {
- return columnIndex;
+ public ImmutableBytesWritable getDictionary() {
+ return dictionary;
}
public ImmutableBytesWritable getKey() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
index 53a5487..eb65ca8 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
@@ -18,14 +18,10 @@
package org.apache.kylin.invertedindex.tools;
-import java.io.IOException;
-import java.util.Iterator;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.kylin.common.KylinConfig;
@@ -37,6 +33,10 @@ import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.invertedindex.model.KeyValuePair;
+
+import java.io.IOException;
+import java.util.Iterator;
/**
* @author yangli9
@@ -65,18 +65,17 @@ public class IICLI {
System.out.println("Total " + count + " records");
}
- public static Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> readSequenceKVs(
+ public static Iterable<KeyValuePair> readSequenceKVs(
Configuration hconf, String path) throws IOException {
final Reader reader = new Reader(hconf,
SequenceFile.Reader.file(new Path(path)));
- return new Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>() {
+ return new Iterable<KeyValuePair>() {
@Override
- public Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> iterator() {
- return new Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>() {
+ public Iterator<KeyValuePair> iterator() {
+ return new Iterator<KeyValuePair>() {
ImmutableBytesWritable k = new ImmutableBytesWritable();
ImmutableBytesWritable v = new ImmutableBytesWritable();
- Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair = new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(
- k, v);
+ KeyValuePair pair = new KeyValuePair(k, v);
@Override
public boolean hasNext() {
@@ -94,7 +93,7 @@ public class IICLI {
}
@Override
- public Pair<ImmutableBytesWritable, ImmutableBytesWritable> next() {
+ public KeyValuePair next() {
return pair;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index 4b417e8..f040820 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -30,6 +30,7 @@ import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.invertedindex.model.KeyValuePair;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -136,8 +137,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
System.out.println(slices.size() + " slices");
IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = encodeKVs(
- codec, slices);
+ List<KeyValuePair> kvs = encodeKVs(codec, slices);
System.out.println(kvs.size() + " KV pairs");
List<Slice> slicesCopy = decodeKVs(codec, kvs);
@@ -198,10 +198,10 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
return slices;
}
- private List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKVs(
+ private List<KeyValuePair> encodeKVs(
IIKeyValueCodec codec, List<Slice> slices) {
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = Lists
+ List<KeyValuePair> kvs = Lists
.newArrayList();
for (Slice slice : slices) {
kvs.addAll(codec.encodeKeyValue(slice));
@@ -210,7 +210,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
}
private List<Slice> decodeKVs(IIKeyValueCodec codec,
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
+ List<KeyValuePair> kvs) {
List<Slice> slices = Lists.newArrayList();
for (Slice slice : codec.decodeKeyValue(kvs)) {
slices.add(slice);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index df26e41..6bcae0d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.invertedindex.model.KeyValuePair;
import java.io.Closeable;
import java.io.IOException;
@@ -33,28 +34,28 @@ import java.util.List;
/**
* Created by honma on 11/10/14.
*/
-public class HbaseServerKVIterator implements Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>, Closeable {
+public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable {
- private RegionScanner innerScaner;
+ private RegionScanner innerScanner;
List<Cell> results = new ArrayList<Cell>();
- public HbaseServerKVIterator(RegionScanner innerScaner) {
- this.innerScaner = innerScaner;
+ public HbaseServerKVIterator(RegionScanner innerScanner) {
+ this.innerScanner = innerScanner;
}
@Override
public void close() throws IOException {
- IOUtils.closeQuietly(this.innerScaner);
+ IOUtils.closeQuietly(this.innerScanner);
}
@Override
- public Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> iterator() {
- return new Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>() {
+ public Iterator<KeyValuePair> iterator() {
+ return new Iterator<KeyValuePair>() {
ImmutableBytesWritable key = new ImmutableBytesWritable();
ImmutableBytesWritable value = new ImmutableBytesWritable();
- Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair = new Pair<>(key, value);
+ KeyValuePair pair = new KeyValuePair(key, value);
private boolean hasMore = true;
@@ -64,10 +65,10 @@ public class HbaseServerKVIterator implements Iterable<Pair<ImmutableBytesWritab
}
@Override
- public Pair<ImmutableBytesWritable, ImmutableBytesWritable> next() {
+ public KeyValuePair next() {
if (hasNext()) {
try {
- hasMore = innerScaner.nextRaw(results);
+ hasMore = innerScanner.nextRaw(results);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
index 28371a3..af6cf38 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
@@ -6,7 +6,7 @@ import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
-import java.util.List;
+import java.util.Map;
/**
* Created by Hongbin Ma(Binmahone) on 3/3/15.
@@ -14,11 +14,11 @@ import java.util.List;
public class LocalDictionary implements IDictionaryAware {
private CoprocessorRowType type;
- private List<Dictionary<?>> dicts;
+ private Map<Integer, Dictionary<?>> colDictMap;
private TableRecordInfoDigest recordInfo;
- public LocalDictionary(List<Dictionary<?>> dicts, CoprocessorRowType type, TableRecordInfoDigest recordInfo) {
- this.dicts = dicts;
+ public LocalDictionary(Map<Integer, Dictionary<?>> colDictMap, CoprocessorRowType type, TableRecordInfoDigest recordInfo) {
+ this.colDictMap = colDictMap;
this.type = type;
this.recordInfo = recordInfo;
}
@@ -30,6 +30,6 @@ public class LocalDictionary implements IDictionaryAware {
@Override
public Dictionary<?> getDictionary(TblColRef col) {
- return this.dicts.get(type.getColIndexByTblColRef(col));
+ return this.colDictMap.get(type.getColIndexByTblColRef(col));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af6372de/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 6339fe4..36536bb 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -50,6 +50,7 @@ import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.invertedindex.model.KeyValuePair;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -164,9 +165,9 @@ public class IIStreamBuilder extends StreamBuilder {
private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
try {
List<Put> data = Lists.newArrayList();
- for (Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair : codec.encodeKeyValue(slice)) {
- final byte[] key = pair.getFirst().get();
- final byte[] value = pair.getSecond().get();
+ for (KeyValuePair pair : codec.encodeKeyValue(slice)) {
+ final byte[] key = pair.getKey().get();
+ final byte[] value = pair.getValue().get();
Put put = new Put(key);
put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
//dictionary
[05/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2209fde6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2209fde6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2209fde6
Branch: refs/heads/streaming-localdict
Commit: 2209fde6a09fe36cbe7b77bd32f52702c1388618
Parents: 37f9a73 d2dc5cd
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Mar 10 16:56:56 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Mar 10 16:56:56 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 18 +++
.../org/apache/kylin/cube/model/CubeDesc.java | 4 -
docs/Installation/install_and_run.md | 7 +-
docs/website/download/index.md | 19 ++-
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 156 +++++++++++++++++++
.../kylin/metadata/filter/CaseTupleFilter.java | 18 +--
.../metadata/filter/ColumnTupleFilter.java | 25 +--
.../metadata/filter/CompareTupleFilter.java | 72 ++++-----
.../metadata/filter/ConstantTupleFilter.java | 23 ++-
.../metadata/filter/DynamicTupleFilter.java | 9 +-
.../metadata/filter/ExtractTupleFilter.java | 20 ++-
.../kylin/metadata/filter/ICodeSystem.java | 23 +++
.../metadata/filter/IEvaluatableTuple.java | 9 ++
.../metadata/filter/LogicalTupleFilter.java | 28 ++--
.../kylin/metadata/filter/StringCodeSystem.java | 40 +++++
.../kylin/metadata/filter/TupleFilter.java | 12 +-
.../metadata/filter/TupleFilterSerializer.java | 26 ++--
.../metadata/measure/BigDecimalSerializer.java | 18 ++-
.../metadata/measure/DataTypeSerializer.java | 83 ++++++++++
.../metadata/measure/DoubleSerializer.java | 7 +-
.../kylin/metadata/measure/HLLCSerializer.java | 7 +-
.../kylin/metadata/measure/LongSerializer.java | 14 +-
.../kylin/metadata/measure/MeasureCodec.java | 8 +-
.../metadata/measure/MeasureSerializer.java | 68 --------
.../kylin/metadata/model/DataModelDesc.java | 10 +-
.../kylin/metadata/model/DatabaseDesc.java | 2 +-
.../kylin/metadata/model/PartitionDesc.java | 12 +-
.../apache/kylin/metadata/model/TableDesc.java | 3 +-
.../metadata/tool/HiveSourceTableLoader.java | 13 +-
.../org/apache/kylin/metadata/tuple/ITuple.java | 5 +-
.../kylin/query/enumerator/CubeEnumerator.java | 16 +-
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/test/KylinQueryTest.java | 18 +--
.../storage/filter/BitMapFilterEvaluator.java | 14 +-
.../kylin/storage/gridtable/GTBuilder.java | 68 ++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 52 +++++++
.../kylin/storage/gridtable/GTRecord.java | 15 ++
.../kylin/storage/gridtable/GTRowBlock.java | 34 ++++
.../storage/gridtable/GTRowBlockIndex.java | 5 +
.../apache/kylin/storage/gridtable/GTStore.java | 35 +++++
.../kylin/storage/gridtable/GridTable.java | 19 +++
.../kylin/storage/gridtable/IKVStoreReader.java | 11 ++
.../kylin/storage/gridtable/IKVStoreWriter.java | 11 ++
.../kylin/storage/hbase/CubeStorageEngine.java | 5 +-
.../storage/hbase/DerivedFilterTranslator.java | 35 +----
.../hbase/coprocessor/CoprocessorFilter.java | 16 +-
.../hbase/coprocessor/DictCodeSystem.java | 45 ++++++
.../hbase/coprocessor/FilterDecorator.java | 37 ++---
.../coprocessor/observer/ObserverTuple.java | 33 +---
.../filter/BitMapFilterEvaluatorTest.java | 1 -
.../kylin/storage/filter/FilterBaseTest.java | 4 +-
.../storage/filter/FilterEvaluateTest.java | 36 ++---
.../storage/filter/FilterSerializeTest.java | 48 +++---
webapp/app/js/controllers/cubeEdit.js | 90 +++++++----
webapp/app/js/controllers/cubeModel.js | 35 ++++-
webapp/app/js/controllers/cubeSchema.js | 7 -
webapp/app/js/model/jobListModel.js | 11 --
webapp/app/js/model/metaModel.js | 2 +-
.../cubeDesigner/advanced_settings.html | 1 +
.../app/partials/cubeDesigner/dimensions.html | 1 -
.../app/partials/cubeDesigner/incremental.html | 10 +-
62 files changed, 1023 insertions(+), 459 deletions(-)
----------------------------------------------------------------------
[20/50] [abbrv] incubator-kylin git commit: Merge branch 'staging'
into streaming
Posted by li...@apache.org.
Merge branch 'staging' into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0b71528f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0b71528f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0b71528f
Branch: refs/heads/streaming-localdict
Commit: 0b71528f6fe569fc12d1a609ac52240f90236b63
Parents: b25e227 54dc31f
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 14:02:33 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 14:02:33 2015 +0800
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 1 +
.../kylin/job/hadoop/AbstractHadoopJob.java | 65 ++++++++++++++------
.../apache/kylin/metadata/MetadataManager.java | 1 +
4 files changed, 48 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
[35/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5b5cb31f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5b5cb31f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5b5cb31f
Branch: refs/heads/streaming-localdict
Commit: 5b5cb31fbc928c65dd35213794a84f1b2c790f66
Parents: b36d1e8
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 21:02:01 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 21:02:01 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/index/SliceBuilder.java | 4 +
.../invertedindex/index/TableRecordInfo.java | 10 ++-
.../invertedindex/model/IIKeyValueCodec.java | 52 ++++---------
.../apache/kylin/invertedindex/tools/IICLI.java | 2 +-
.../invertedindex/InvertedIndexLocalTest.java | 79 +++++++++++++++-----
.../invertedindex/InvertedIndexReducer.java | 2 +-
.../endpoint/HbaseServerKVIterator.java | 7 ++
.../hbase/coprocessor/endpoint/IIEndpoint.java | 2 +-
.../storage/hbase/InvertedIndexHBaseTest.java | 4 +-
.../invertedindex/IIStreamBuilder.java | 12 +--
10 files changed, 103 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index 72050a1..07cabf1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -61,6 +61,10 @@ public class SliceBuilder {
// reset for next slice
nRecords = 0;
containers = new ColumnValueContainer[nColumns];
+ for (int i : info.getDescriptor().getBitmapColumns()) {
+ containers[i] = new CompressedValueContainer(info.getDigest(), i,
+ nRecordsCap);
+ }
for (int i : info.getDescriptor().getValueColumns()) {
containers[i] = new CompressedValueContainer(info.getDigest(), i,
nRecordsCap);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 3e72f43..c178742 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -73,9 +73,17 @@ public class TableRecordInfo {
digest = createDigest(dictionaryMap, measureCodecMap);
}
- public TableRecordInfo(IIDesc desc, Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+ public TableRecordInfo(IIDesc desc, Map<Integer, Dictionary<?>> dictionaryMap) {
this.desc = desc;
this.dictionaryMap = dictionaryMap;
+ Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap = Maps.newHashMap();
+ int index = 0;
+ for (TblColRef tblColRef : desc.listAllColumns()) {
+ ColumnDesc col = tblColRef.getColumn();
+ if (desc.isMetricsCol(index++)) {
+ measureCodecMap.put(tblColRef, FixedLenMeasureCodec.get(col.getType()));
+ }
+ }
this.digest = createDigest(dictionaryMap, measureCodecMap);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 31fa48a..d7372bc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -24,10 +24,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.index.ColumnValueContainer;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import org.apache.kylin.invertedindex.index.*;
import java.io.*;
import java.util.*;
@@ -40,8 +37,10 @@ public class IIKeyValueCodec implements KeyValueCodec {
public static final int SHARD_LEN = 2;
public static final int TIMEPART_LEN = 8;
public static final int COLNO_LEN = 2;
+ private final TableRecordInfoDigest digest;
- public IIKeyValueCodec() {
+ public IIKeyValueCodec(TableRecordInfoDigest digest) {
+ this.digest = digest;
}
@Override
@@ -114,40 +113,13 @@ public class IIKeyValueCodec implements KeyValueCodec {
@Override
public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
- return new Decoder(kvs);
+ return new Decoder(kvs, digest);
}
- private static class SliceIterator implements Iterator<Slice> {
-
- private final Iterator<IIRow> rowIterator;
-
- public SliceIterator(Iterator<IIRow> rowIterator) {
- this.rowIterator = rowIterator;
- }
-
- @Override
- public boolean hasNext() {
- if (!rowIterator.hasNext()) {
- return false;
- }
- return false;
- }
-
- @Override
- public Slice next() {
- return null;
- }
-
- @Override
- public void remove() {
-
- }
- }
-
private static class Decoder implements Iterable<Slice> {
- TableRecordInfoDigest info;
- Iterator<IIRow> iterator;
+ private final TableRecordInfoDigest digest;
+ Iterator<IIRow> iterator;
Slice slice = null;
short curShard = Short.MIN_VALUE;
@@ -159,7 +131,8 @@ public class IIKeyValueCodec implements KeyValueCodec {
ColumnValueContainer[] containers = null;
Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
- Decoder(Iterable<IIRow> kvs) {
+ Decoder(Iterable<IIRow> kvs, TableRecordInfoDigest digest) {
+ this.digest = digest;
this.iterator = kvs.iterator();
}
@@ -184,6 +157,9 @@ public class IIKeyValueCodec implements KeyValueCodec {
containers[curCol].append(buffer);
}
localDictionaries.put(curCol, dictionary);
+ if (localDictionaries.size() < digest.getColumnCount()) {
+ continue;
+ }
if (curShard != lastShard
|| curSliceTimestamp != lastSliceTimestamp) {
@@ -218,7 +194,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
private void makeNext() {
if (containers != null) {
- slice = new Slice(info, lastShard, lastSliceTimestamp,
+ slice = new Slice(digest, lastShard, lastSliceTimestamp,
containers);
slice.setLocalDictionaries(Maps.newHashMap(localDictionaries));
}
@@ -230,7 +206,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
private void addContainer(int col, ColumnValueContainer c) {
if (containers == null) {
- containers = new ColumnValueContainer[info.getColumnCount()];
+ containers = new ColumnValueContainer[digest.getColumnCount()];
}
containers[col] = c;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
index aecc37d..068af07 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
@@ -54,7 +54,7 @@ public class IICLI {
System.out.println("Reading from " + path + " ...");
TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment());
- IIKeyValueCodec codec = new IIKeyValueCodec();
+ IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
int count = 0;
for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) {
for (RawTableRecord rec : slice) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index b152cdd..c49a8dd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -18,29 +18,31 @@
package org.apache.kylin.invertedindex.invertedindex;
-import com.google.common.collect.Lists;
+import com.google.common.base.Function;
+import com.google.common.collect.*;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.invertedindex.model.IIRow;
+import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import javax.annotation.Nullable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
import static org.junit.Assert.assertEquals;
@@ -48,12 +50,28 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
IIInstance ii;
TableRecordInfo info;
+ List<String> lines;
+ private Map<Integer, Dictionary<?>> dictionaryMap;
- @Before
+ @Before
public void setUp() throws Exception {
this.createTestMetadata();
this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
- this.info = new TableRecordInfo(ii.getFirstSegment());
+
+ File file = new File(LOCALMETA_TEST_DATA,
+ "data/flatten_data_for_ii.csv");
+ FileInputStream in = new FileInputStream(file);
+ this.lines = IOUtils.readLines(in, "UTF-8");
+ in.close();
+
+ dictionaryMap = buildDictionary(Lists.transform(lines, new Function<String, List<String>>() {
+ @Nullable
+ @Override
+ public List<String> apply(@Nullable String input) {
+ return Lists.newArrayList(input.split(","));
+ }
+ }), ii.getDescriptor());
+ this.info = new TableRecordInfo(ii.getDescriptor(), dictionaryMap);
}
@After
@@ -108,7 +126,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
List<Slice> slices = buildTimeSlices(records);
System.out.println(slices.size() + " slices");
- IIKeyValueCodec codec = new IIKeyValueCodec();
+ IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
List<IIRow> kvs = encodeKVs(codec, slices);
System.out.println(kvs.size() + " KV pairs");
@@ -121,13 +139,34 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
dump(recordsCopy);
}
- private List<TableRecord> loadRecordsSorted() throws IOException {
- File file = new File(LOCALMETA_TEST_DATA,
- "data/flatten_data_for_ii.csv");
- FileInputStream in = new FileInputStream(file);
- List<String> lines = IOUtils.readLines(in, "UTF-8");
- in.close();
+ private Map<Integer, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+ Set<TblColRef> dimensionColumns = Sets.newHashSet();
+ for (int i = 0; i < desc.listAllColumns().size(); i++) {
+ if (!desc.isMetricsCol(i)) {
+ dimensionColumns.add(desc.listAllColumns().get(i));
+ }
+ }
+ for (List<String> row : table) {
+ for (int i = 0; i < row.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(desc.listAllColumns().get(i), cell);
+ }
+ }
+ Map<Integer, Dictionary<?>> result = Maps.newHashMap();
+ for (TblColRef tblColRef : valueMap.keys()) {
+ result.put(desc.findColumn(tblColRef), DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ return input.getBytes();
+ }
+ }), tblColRef.getType()));
+ }
+ return result;
+ }
+ private List<TableRecord> loadRecordsSorted() throws IOException {
List<TableRecord> records = Lists.newArrayList();
for (String line : lines) {
String[] fields = line.split(",");
@@ -159,12 +198,18 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
List<Slice> slices = Lists.newArrayList();
for (TableRecord rec : records) {
+ //here assume there less records than slice size for each shard
Slice slice = builder.append(rec);
- if (slice != null)
- slices.add(slice);
+ if (slice != null) {
+ slice.setLocalDictionaries(dictionaryMap);
+ slices.add(slice);
+ }
}
List<Slice> finals = builder.close();
- slices.addAll(finals);
+ for (Slice slice : finals) {
+ slice.setLocalDictionaries(dictionaryMap);
+ }
+ slices.addAll(finals);
Collections.sort(slices);
return slices;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
index 285b058..a1c0811 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
@@ -60,7 +60,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
info = new TableRecordInfo(seg);
rec = info.createTableRecord();
builder = null;
- kv = new IIKeyValueCodec();
+ kv = new IIKeyValueCodec(info.getDigest());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index 3dbc8ea..d0cfeb0 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -23,6 +23,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIRow;
import org.slf4j.Logger;
@@ -39,6 +40,7 @@ import java.util.List;
*/
public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
+ private TableRecordInfoDigest digest;
private RegionScanner innerScanner;
private Logger logger = LoggerFactory.getLogger(HbaseServerKVIterator.class);
@@ -46,6 +48,11 @@ public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
this.innerScanner = innerScanner;
}
+ public HbaseServerKVIterator(RegionScanner innerScanner, TableRecordInfoDigest digest) {
+ this.innerScanner = innerScanner;
+ this.digest = digest;
+ }
+
@Override
public void close() throws IOException {
IOUtils.closeQuietly(this.innerScanner);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index a20104d..935acb7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -90,7 +90,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
region.startRegionOperation();
synchronized (innerScanner) {
- IIKeyValueCodec codec = new IIKeyValueCodec();
+ IIKeyValueCodec codec = new IIKeyValueCodec(tableRecordInfoDigest);
//TODO pass projector to codec to skip loading columns
Iterable<Slice> slices = codec.decodeKeyValue(new HbaseServerKVIterator(innerScanner));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index 96c363c..548224c 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -69,7 +69,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
hconn = HConnectionManager.createConnection(hconf);
- this.info = new TableRecordInfo(seg.getIIDesc(), Collections.<Integer, Dictionary<?>>emptyMap(), Collections.<TblColRef, FixedLenMeasureCodec<?>>emptyMap());
+ this.info = new TableRecordInfo(seg.getIIDesc(), null);
}
@After
@@ -81,7 +81,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
public void testLoad() throws Exception {
String tableName = seg.getStorageLocationIdentifier();
- IIKeyValueCodec codec = new IIKeyValueCodec();
+ IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
List<Slice> slices = Lists.newArrayList();
HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b5cb31f/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 80ff21b..90d6eaf 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -104,18 +104,10 @@ public class IIStreamBuilder extends StreamBuilder {
}
});
final Map<Integer, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
- Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap = Maps.newHashMap();
- int index = 0;
- for (TblColRef tblColRef : desc.listAllColumns()) {
- ColumnDesc col = tblColRef.getColumn();
- if (desc.isMetricsCol(index++)) {
- measureCodecMap.put(tblColRef, FixedLenMeasureCodec.get(col.getType()));
- }
- }
- TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap, measureCodecMap);
+ TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap);
SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo, dictionaryMap);
- loadToHBase(hTable, slice, new IIKeyValueCodec());
+ loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()));
submitOffset();
stopwatch.stop();
logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS);
[36/50] [abbrv] incubator-kylin git commit: KYLIN-638 kylin.sh stop
not working
Posted by li...@apache.org.
KYLIN-638 kylin.sh stop not working
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/624c4347
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/624c4347
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/624c4347
Branch: refs/heads/streaming-localdict
Commit: 624c4347feee9e3bcfca5ae7e8538abe82fbf152
Parents: 0b13251
Author: honma <ho...@ebay.com>
Authored: Thu Mar 12 16:36:40 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 12 16:36:40 2015 -0700
----------------------------------------------------------------------
bin/kylin.sh | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/624c4347/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 7981bb8..7941093 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -1,9 +1,11 @@
#!/bin/sh
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
if [ $1 == "start" ]
then
- dir=$(dirname ${0})
- source ${dir}/check-env.sh
+
tomcat_root=${dir}/../tomcat
export tomcat_root
[34/50] [abbrv] incubator-kylin git commit: fix compile issue
Posted by li...@apache.org.
fix compile issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b36d1e87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b36d1e87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b36d1e87
Branch: refs/heads/streaming-localdict
Commit: b36d1e87543cd9e4b55ad56dde4ed410656e4c39
Parents: bb67362
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 18:50:45 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 18:50:45 2015 +0800
----------------------------------------------------------------------
.../invertedindex/model/IIKeyValueCodec.java | 61 +++++++++++++-------
.../storage/hbase/InvertedIndexHBaseTest.java | 4 +-
.../invertedindex/IIStreamBuilder.java | 8 ---
3 files changed, 42 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b36d1e87/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 6955229..31fa48a 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -51,23 +51,47 @@ public class IIKeyValueCodec implements KeyValueCodec {
ColumnValueContainer[] containers = slice.getColumnValueContainers();
for (int col = 0; col < containers.length; col++) {
if (containers[col] instanceof CompressedValueContainer) {
- result.add(collectKeyValues(slice, col, (CompressedValueContainer) containers[col]));
- } else {
- throw new IllegalArgumentException("Unknown container class "
+ final IIRow row = collectKeyValues(slice, col, (CompressedValueContainer) containers[col]);
+ result.add(row);
+ } else {
+ throw new IllegalArgumentException("Unknown container class "
+ containers[col].getClass());
- }
- }
+ }
+ }
return result;
}
- private IIRow collectKeyValues(Slice slice,
- int col,
- CompressedValueContainer container) {
+ private IIRow collectKeyValues(Slice slice, int col, CompressedValueContainer container) {
ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col);
ImmutableBytesWritable value = container.toBytes();
- return new IIRow(key, value, null);
+ final Dictionary<?> dictionary = slice.getLocalDictionaries().get(col);
+ return new IIRow(key, value, serialize(dictionary));
}
+ private static Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
+ try {
+ final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
+ final String type = dataInputStream.readUTF();
+ final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
+ dictionary.readFields(dataInputStream);
+ return dictionary;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static ImmutableBytesWritable serialize(Dictionary<?> dict) {
+ try {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ out.writeUTF(dict.getClass().getName());
+ dict.write(out);
+ return new ImmutableBytesWritable(baos.toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) {
byte[] bytes = new byte[20];
int len = encodeKey(shard, timestamp, col, bytes, 0);
@@ -139,18 +163,6 @@ public class IIKeyValueCodec implements KeyValueCodec {
this.iterator = kvs.iterator();
}
- private Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
- try {
- final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
- final String type = dataInputStream.readUTF();
- final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
- dictionary.readFields(dataInputStream);
- return dictionary;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
private void goToNext() {
if (slice != null) { // was not fetched
return;
@@ -164,6 +176,13 @@ public class IIKeyValueCodec implements KeyValueCodec {
decodeKey(k);
final Dictionary<?> dictionary = deserialize(kv.getDictionary());
addContainer(curCol, new CompressedValueContainer(dictionary.getSizeOfId(), (dictionary.getMaxId() - dictionary.getMinId() + 1), (dictionary.getMaxId() - dictionary.getMinId() + 1)));
+ byte[] bytes = new byte[dictionary.getSizeOfValue()];
+ ImmutableBytesWritable buffer = new ImmutableBytesWritable(bytes);
+ for (int i = dictionary.getMinId(); i <= dictionary.getMaxId(); ++i) {
+ final int length = dictionary.getValueBytesFromId(i, bytes, 0);
+ buffer.set(bytes, 0, length);
+ containers[curCol].append(buffer);
+ }
localDictionaries.put(curCol, dictionary);
if (curShard != lastShard
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b36d1e87/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index 3ac832a..96c363c 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -69,7 +69,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
hconn = HConnectionManager.createConnection(hconf);
- this.info = new TableRecordInfo(seg.getIIDesc(), Collections.<TblColRef, Dictionary<?>>emptyMap(), Collections.<TblColRef, FixedLenMeasureCodec<?>>emptyMap());
+ this.info = new TableRecordInfo(seg.getIIDesc(), Collections.<Integer, Dictionary<?>>emptyMap(), Collections.<TblColRef, FixedLenMeasureCodec<?>>emptyMap());
}
@After
@@ -81,7 +81,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
public void testLoad() throws Exception {
String tableName = seg.getStorageLocationIdentifier();
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+ IIKeyValueCodec codec = new IIKeyValueCodec();
List<Slice> slices = Lists.newArrayList();
HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b36d1e87/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 10d212a..80ff21b 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -165,14 +165,6 @@ public class IIStreamBuilder extends StreamBuilder {
return slice;
}
- private byte[] getDictBytes(Dictionary<?> dict) throws IOException {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos);
- out.writeUTF(dict.getClass().getName());
- dict.write(out);
- return baos.toByteArray();
- }
-
private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
try {
List<Put> data = Lists.newArrayList();
[30/50] [abbrv] incubator-kylin git commit: fix ut
Posted by li...@apache.org.
fix ut
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a1b0177a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a1b0177a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a1b0177a
Branch: refs/heads/streaming-localdict
Commit: a1b0177a668a1fde1754387931407df10fb0fef0
Parents: 8be38e7
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 15:02:02 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 15:02:02 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/index/TableRecordInfo.java | 12 +++++++++---
.../kylin/storage/hbase/HBaseClientKVIterator.java | 4 +---
.../kylin/storage/hbase/InvertedIndexHBaseTest.java | 6 +++++-
.../kylin/streaming/invertedindex/IIStreamBuilder.java | 6 ++++++
4 files changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 5535fcc..0cfb05e 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -92,10 +92,16 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- lengths[i] = measureCodecMap.get(tblColRef).getLength();
+ final FixedLenMeasureCodec<?> fixedLenMeasureCodec = measureCodecMap.get(tblColRef);
+ if (fixedLenMeasureCodec != null) {
+ lengths[i] = fixedLenMeasureCodec.getLength();
+ }
} else {
- lengths[i] = dictionaryMap.get(tblColRef).getSizeOfId();
- dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+ final Dictionary<?> dictionary = dictionaryMap.get(tblColRef);
+ if (dictionary != null) {
+ lengths[i] = dictionary.getSizeOfId();
+ dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+ }
}
}
// offsets
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 339612d..29b2bcf 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -40,7 +40,6 @@ import org.apache.kylin.invertedindex.model.KeyValuePair;
public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable {
byte[] family;
- byte[] qualifier;
HTableInterface table;
ResultScanner scanner;
@@ -48,10 +47,9 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family, byte[] qualifier) throws IOException {
this.family = family;
- this.qualifier = qualifier;
this.table = hconn.getTable(tableName);
- this.scanner = table.getScanner(family, qualifier);
+ this.scanner = table.getScanner(family);
this.iterator = scanner.iterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index f38ed49..3ac832a 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -18,11 +18,15 @@
package org.apache.kylin.storage.hbase;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -65,7 +69,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
hconn = HConnectionManager.createConnection(hconf);
- this.info = new TableRecordInfo(seg);
+ this.info = new TableRecordInfo(seg.getIIDesc(), Collections.<TblColRef, Dictionary<?>>emptyMap(), Collections.<TblColRef, FixedLenMeasureCodec<?>>emptyMap());
}
@After
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1b0177a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index a4c7167..5ed48cf 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -35,6 +35,7 @@
package org.apache.kylin.streaming.invertedindex;
import com.google.common.base.Function;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.*;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -67,6 +68,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
/**
* Created by qianzhou on 3/3/15.
@@ -93,6 +95,8 @@ public class IIStreamBuilder extends StreamBuilder {
@Override
protected void build(List<Stream> streamsToBuild) throws IOException {
+ logger.info("stream build start, size:" + streamsToBuild.size());
+ Stopwatch stopwatch = new Stopwatch().start();
List<List<String>> table = Lists.transform(streamsToBuild, new Function<Stream, List<String>>() {
@Nullable
@Override
@@ -114,6 +118,8 @@ public class IIStreamBuilder extends StreamBuilder {
final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()), desc.listAllColumns(), dictionaryMap);
submitOffset();
+ stopwatch.stop();
+ logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS);
}
private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
[11/50] [abbrv] incubator-kylin git commit: Merge pull request #448
from janzhongi/streaming
Posted by li...@apache.org.
Merge pull request #448 from janzhongi/streaming
merge staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b25e227b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b25e227b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b25e227b
Branch: refs/heads/streaming-localdict
Commit: b25e227be92ed437d4ddafac40ae4da44579de3a
Parents: bb83ca1 e3c5072
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 18:22:07 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 18:22:07 2015 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
14 files changed, 115 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
[38/50] [abbrv] incubator-kylin git commit: Merge remote-tracking
branch 'origin/streaming' into streaming-localdict
Posted by li...@apache.org.
Merge remote-tracking branch 'origin/streaming' into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f6e73238
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f6e73238
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f6e73238
Branch: refs/heads/streaming-localdict
Commit: f6e73238a77d40f865eaa9f85c1ec0474126bda9
Parents: 5781a0a 624c434
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 13:04:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 13:04:16 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 6 ++++--
.../main/java/org/apache/kylin/streaming/KafkaConsumer.java | 4 +++-
.../java/org/apache/kylin/streaming/KafkaConsumerTest.java | 2 ++
3 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f6e73238/bin/kylin.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f6e73238/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f6e73238/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
[48/50] [abbrv] incubator-kylin git commit: KYLIN-625, draft impl,
ready for very first test
Posted by li...@apache.org.
KYLIN-625, draft impl, ready for very first test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4e290fb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4e290fb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4e290fb0
Branch: refs/heads/streaming-localdict
Commit: 4e290fb07a842a874421f0ce7618d4469dbd95d0
Parents: 5059723
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 13 18:41:55 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 13 18:41:55 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 72 ++++++-
.../org/apache/kylin/common/util/BytesUtil.java | 21 +++
.../kylin/dict/lookup/LookupBytesTable.java | 2 +-
.../cube/FactDistinctColumnsCombiner.java | 2 +-
.../hadoop/cube/FactDistinctColumnsReducer.java | 2 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 2 +-
.../IIDistinctColumnsCombiner.java | 2 +-
.../invertedindex/IIDistinctColumnsReducer.java | 2 +-
.../kylin/metadata/filter/CaseTupleFilter.java | 7 +-
.../metadata/filter/ColumnTupleFilter.java | 7 +-
.../metadata/filter/CompareTupleFilter.java | 7 +-
.../metadata/filter/ConstantTupleFilter.java | 7 +-
.../metadata/filter/DynamicTupleFilter.java | 7 +-
.../metadata/filter/ExtractTupleFilter.java | 7 +-
.../metadata/filter/IFilterCodeSystem.java | 30 +++
.../metadata/filter/LogicalTupleFilter.java | 13 +-
.../kylin/metadata/filter/StringCodeSystem.java | 15 +-
.../kylin/metadata/filter/TupleFilter.java | 7 +-
.../metadata/filter/TupleFilterSerializer.java | 11 +-
.../metadata/measure/BigDecimalSerializer.java | 75 --------
.../metadata/measure/DataTypeSerializer.java | 83 --------
.../metadata/measure/DoubleSerializer.java | 60 ------
.../kylin/metadata/measure/HLLCSerializer.java | 72 -------
.../kylin/metadata/measure/LongSerializer.java | 67 -------
.../kylin/metadata/measure/MeasureCodec.java | 2 +-
.../serializer/BigDecimalSerializer.java | 75 ++++++++
.../metadata/serializer/DataTypeSerializer.java | 87 +++++++++
.../metadata/serializer/DoubleSerializer.java | 60 ++++++
.../metadata/serializer/HLLCSerializer.java | 72 +++++++
.../metadata/serializer/LongSerializer.java | 67 +++++++
.../metadata/serializer/StringSerializer.java | 30 +++
.../kylin/metadata/tuple/ICodeSystem.java | 30 ---
.../storage/gridtable/GTAggregateScanner.java | 144 ++++++++++++++
.../kylin/storage/gridtable/GTBuilder.java | 32 +---
.../kylin/storage/gridtable/GTCodeSystem.java | 76 ++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 188 ++++++++++++++++---
.../kylin/storage/gridtable/GTRawScanner.java | 169 +++++++++++++++++
.../kylin/storage/gridtable/GTRecord.java | 131 ++++++++++++-
.../kylin/storage/gridtable/GTRowBlock.java | 103 ++++++++--
.../apache/kylin/storage/gridtable/GTStore.java | 35 ----
.../kylin/storage/gridtable/GridTable.java | 25 ++-
.../kylin/storage/gridtable/IGTCodeSystem.java | 25 +++
.../kylin/storage/gridtable/IGTScanner.java | 7 +
.../kylin/storage/gridtable/IGTStore.java | 37 ++++
.../kylin/storage/gridtable/IKVStoreReader.java | 11 --
.../kylin/storage/gridtable/IKVStoreWriter.java | 11 --
.../gridtable/memstore/GTSimpleMemStore.java | 96 ++++++++++
.../hbase/coprocessor/DictCodeSystem.java | 19 +-
.../kylin/storage/filter/FilterBaseTest.java | 3 +-
49 files changed, 1521 insertions(+), 594 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index df6f01b..3d3a291 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -18,7 +18,7 @@
package org.apache.kylin.common.util;
-import java.util.Arrays;
+import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.util.Bytes;
@@ -27,15 +27,65 @@ import org.apache.hadoop.hbase.util.Bytes;
*/
public class ByteArray implements Comparable<ByteArray> {
- public byte[] data;
+ public static ByteArray allocate(int length) {
+ return new ByteArray(new byte[length]);
+ }
+
+ // ============================================================================
+
+ private byte[] data;
+ private int offset;
+ private int length;
+
+ public ByteArray() {
+ set(null, 0, 0);
+ }
public ByteArray(byte[] data) {
- this.data = data;
+ set(data, 0, data.length);
+ }
+
+ public ByteArray(byte[] data, int offset, int length) {
+ set(data, offset, length);
+ }
+
+ public byte[] array() {
+ return data;
+ }
+
+ public int offset() {
+ return offset;
+ }
+
+ public int length() {
+ return length;
+ }
+
+ public void set(byte[] array) {
+ set(array, 0, array.length);
+ }
+
+ public void set(byte[] array, int offset, int length) {
+ this.data = array;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public void set(ByteArray o) {
+ set(o.data, o.offset, o.length);
+ }
+
+ public void setLength(int length) {
+ this.length = length;
+ }
+
+ public ByteBuffer wrapAsBuffer() {
+ return ByteBuffer.wrap(data, offset, length);
}
@Override
public int hashCode() {
- return Bytes.hashCode(data);
+ return Bytes.hashCode(data, offset, length);
}
@Override
@@ -46,14 +96,18 @@ public class ByteArray implements Comparable<ByteArray> {
return false;
if (getClass() != obj.getClass())
return false;
- ByteArray other = (ByteArray) obj;
- if (!Arrays.equals(data, other.data))
- return false;
- return true;
+ ByteArray o = (ByteArray) obj;
+ return Bytes.equals(this.data, this.offset, this.length, o.data, o.offset, o.length);
}
@Override
public int compareTo(ByteArray o) {
- return Bytes.compareTo(this.data, o.data);
+ return Bytes.compareTo(this.data, this.offset, this.length, o.data, o.offset, o.length);
+ }
+
+ @Override
+ public String toString() {
+ return Bytes.toString(data, offset, length);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 2cd8d7d..dbd459d 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -311,6 +311,15 @@ public class BytesUtil {
out.put(array);
}
+ public static void writeByteArray(byte[] array, int offset, int length, ByteBuffer out) {
+ if (array == null) {
+ writeVInt(-1, out);
+ return;
+ }
+ writeVInt(array.length, out);
+ out.put(array, offset, length);
+ }
+
public static byte[] readByteArray(ByteBuffer in) {
int len = readVInt(in);
if (len < 0)
@@ -320,6 +329,18 @@ public class BytesUtil {
in.get(array);
return array;
}
+
+ public static int peekByteArrayLength(ByteBuffer in) {
+ int start = in.position();
+ int arrayLen = readVInt(in);
+ int sizeLen = in.position() - start;
+ in.position(start);
+
+ if (arrayLen < 0)
+ return sizeLen;
+ else
+ return sizeLen + arrayLen;
+ }
public static void writeBooleanArray(boolean[] array, ByteBuffer out) {
if (array == null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
index 0b40be0..6e5f7d9 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
@@ -46,7 +46,7 @@ public class LookupBytesTable extends LookupTable<ByteArray> {
@Override
protected String toString(ByteArray cell) {
- return Bytes.toString(cell.data);
+ return cell.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
index c9a7a34..b3912d9 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
@@ -50,7 +50,7 @@ public class FactDistinctColumnsCombiner extends KylinReducer<ShortWritable, Tex
}
for (ByteArray value : set) {
- outputValue.set(value.data);
+ outputValue.set(value.array(), value.offset(), value.length());
context.write(key, outputValue);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index fdae135..15b4970 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -82,7 +82,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
try {
for (ByteArray value : set) {
- out.write(value.data);
+ out.write(value.array(), value.offset(), value.length());
out.write('\n');
}
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
index 9dda89c..e75457e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
@@ -256,7 +256,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
}
} else {
for (Pair<Integer, Integer> relation : tableJoin.dimTblColAsRowKey) {
- keyBytesBuf[relation.getSecond()] = dimRow[relation.getFirst()].data;
+ keyBytesBuf[relation.getSecond()] = dimRow[relation.getFirst()].array();
filledDimension++;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
index dc04d6e..dcfd6f6 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
@@ -51,7 +51,7 @@ public class IIDistinctColumnsCombiner extends KylinReducer<ShortWritable, Text,
}
for (ByteArray value : set) {
- outputValue.set(value.data);
+ outputValue.set(value.array(), value.offset(), value.length());
context.write(key, outputValue);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
index d8f9543..e5eb65e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
@@ -66,7 +66,7 @@ public class IIDistinctColumnsReducer extends KylinReducer<ShortWritable, Text,
try {
for (ByteArray value : set) {
- out.write(value.data);
+ out.write(value.array(), value.offset(), value.length());
out.write('\n');
}
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 4b85172..e8ccd27 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -65,7 +64,7 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
if (whenFilters.size() != thenFilters.size()) {
elseFilter = whenFilters.remove(whenFilters.size() - 1);
}
@@ -103,12 +102,12 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem<?> cs) {
+ public byte[] serialize(IFilterCodeSystem<?> cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index 75f534d..f689ccb 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -28,7 +28,6 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
@@ -68,7 +67,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
this.tupleValue = tuple.getValue(columnRef);
return true;
}
@@ -85,7 +84,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem<?> cs) {
+ public byte[] serialize(IFilterCodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
@@ -105,7 +104,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
TableDesc table = null;
ColumnDesc column = new ColumnDesc();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 71950ff..899320f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -27,7 +27,6 @@ import java.util.Map;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
@@ -131,7 +130,7 @@ public class CompareTupleFilter extends TupleFilter {
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
@SuppressWarnings({ "unchecked", "rawtypes" })
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
// extract tuple value
Object tupleValue = null;
for (TupleFilter filter : this.children) {
@@ -199,7 +198,7 @@ public class CompareTupleFilter extends TupleFilter {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(IFilterCodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
int size = this.dynamicVariables.size();
BytesUtil.writeVInt(size, buffer);
@@ -213,7 +212,7 @@ public class CompareTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
this.dynamicVariables.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index c92b743..f372b4a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.HashSet;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
@@ -65,7 +64,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
return constantValues.size() > 0;
}
@@ -81,7 +80,7 @@ public class ConstantTupleFilter extends TupleFilter {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
- public byte[] serialize(ICodeSystem cs) {
+ public byte[] serialize(IFilterCodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
int size = this.constantValues.size();
BytesUtil.writeVInt(size, buffer);
@@ -94,7 +93,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
this.constantValues.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index 3d7f65f..a482519 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
@@ -55,7 +54,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
return true;
}
@@ -70,7 +69,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem<?> cs) {
+ public byte[] serialize(IFilterCodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
BytesUtil.writeUTFString(variableName, buffer);
byte[] result = new byte[buffer.position()];
@@ -79,7 +78,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
this.variableName = BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index ad1395c..a7705d3 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -55,7 +54,7 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
// extract tuple value
String extractType = null;
String tupleValue = null;
@@ -115,12 +114,12 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem<?> cs) {
+ public byte[] serialize(IFilterCodeSystem<?> cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/IFilterCodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/IFilterCodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/IFilterCodeSystem.java
new file mode 100644
index 0000000..5382dcc
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/IFilterCodeSystem.java
@@ -0,0 +1,30 @@
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Decides how constant values are coded and compared.
+ *
+ * TupleFilter are involved in both query engine and coprocessor. In query engine, the values are strings.
+ * In coprocessor, the values are dictionary IDs.
+ *
+ * The type parameter is the java type of code, which should be bytes. However some legacy implementation
+ * stores code as String.
+ *
+ * @author yangli9
+ */
+public interface IFilterCodeSystem<T> {
+
+ /** if given code represents the NULL value */
+ boolean isNull(T code);
+
+ /** compare two values by their codes */
+ int compare(T code1, T code2);
+
+ /** write code to buffer */
+ void serialize(T code, ByteBuffer buf);
+
+ /** read code from buffer */
+ T deserialize(ByteBuffer buf);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 0c8f96b..1844392 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -72,7 +71,7 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
switch (this.operator) {
case AND:
return evalAnd(tuple, cs);
@@ -85,7 +84,7 @@ public class LogicalTupleFilter extends TupleFilter {
}
}
- private boolean evalAnd(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ private boolean evalAnd(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
for (TupleFilter filter : this.children) {
if (!filter.evaluate(tuple, cs)) {
return false;
@@ -94,7 +93,7 @@ public class LogicalTupleFilter extends TupleFilter {
return true;
}
- private boolean evalOr(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ private boolean evalOr(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
for (TupleFilter filter : this.children) {
if (filter.evaluate(tuple, cs)) {
return true;
@@ -103,7 +102,7 @@ public class LogicalTupleFilter extends TupleFilter {
return false;
}
- private boolean evalNot(IEvaluatableTuple tuple, ICodeSystem<?> cs) {
+ private boolean evalNot(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
return !this.children.get(0).evaluate(tuple, cs);
}
@@ -118,12 +117,12 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize(ICodeSystem<?> cs) {
+ public byte[] serialize(IFilterCodeSystem<?> cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public void deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
index 5240491..5b0040d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
@@ -3,32 +3,21 @@ package org.apache.kylin.metadata.filter;
import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
/**
* A simple code system where all values are strings and conform to string comparison system.
*
* @author yangli9
*/
-public class StringCodeSystem implements ICodeSystem<String> {
+public class StringCodeSystem implements IFilterCodeSystem<String> {
public static final StringCodeSystem INSTANCE = new StringCodeSystem();
- private StringCodeSystem() {
+ protected StringCodeSystem() {
// singleton
}
@Override
- public String encode(Object value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object decode(String code) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public boolean isNull(String value) {
return value == null;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 75d55c3..5dfce7d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import com.google.common.collect.Maps;
@@ -190,13 +189,13 @@ public abstract class TupleFilter {
public abstract boolean isEvaluable();
- public abstract boolean evaluate(IEvaluatableTuple tuple, ICodeSystem<?> cs);
+ public abstract boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs);
public abstract Collection<?> getValues();
- abstract byte[] serialize(ICodeSystem<?> cs);
+ abstract byte[] serialize(IFilterCodeSystem<?> cs);
- abstract void deserialize(byte[] bytes, ICodeSystem<?> cs);
+ abstract void deserialize(byte[] bytes, IFilterCodeSystem<?> cs);
public static boolean isEvaluableRecursively(TupleFilter filter) {
if (filter == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 5164c5b..f824777 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.Stack;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
/**
* http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
@@ -47,11 +46,11 @@ public class TupleFilterSerializer {
}
}
- public static byte[] serialize(TupleFilter rootFilter, ICodeSystem<?> cs) {
+ public static byte[] serialize(TupleFilter rootFilter, IFilterCodeSystem<?> cs) {
return serialize(rootFilter, null, cs);
}
- public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, ICodeSystem<?> cs) {
+ public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, IFilterCodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
internalSerialize(rootFilter, decorator, buffer, cs);
byte[] result = new byte[buffer.position()];
@@ -59,7 +58,7 @@ public class TupleFilterSerializer {
return result;
}
- private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem<?> cs) {
+ private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, IFilterCodeSystem<?> cs) {
if (decorator != null) { // give decorator a chance to manipulate the
// output filter
filter = decorator.onSerialize(filter);
@@ -84,7 +83,7 @@ public class TupleFilterSerializer {
}
}
- private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem<?> cs) {
+ private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer, IFilterCodeSystem<?> cs) {
if (flag < 0) {
BytesUtil.writeVInt(-1, buffer);
} else {
@@ -96,7 +95,7 @@ public class TupleFilterSerializer {
}
}
- public static TupleFilter deserialize(byte[] bytes, ICodeSystem<?> cs) {
+ public static TupleFilter deserialize(byte[] bytes, IFilterCodeSystem<?> cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
TupleFilter rootFilter = null;
Stack<TupleFilter> parentStack = new Stack<TupleFilter>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
deleted file mode 100644
index 78654ea..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
-
- @Override
- public void serialize(BigDecimal value, ByteBuffer out) {
- byte[] bytes = value.unscaledValue().toByteArray();
-
- BytesUtil.writeVInt(value.scale(), out);
- BytesUtil.writeVInt(bytes.length, out);
- out.put(bytes);
- }
-
- @Override
- public BigDecimal deserialize(ByteBuffer in) {
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
-
- byte[] bytes = new byte[n];
- in.get(bytes);
-
- return new BigDecimal(new BigInteger(bytes), scale);
- }
-
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- @SuppressWarnings("unused")
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
- int len = in.position() - mark + n;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public BigDecimal valueOf(byte[] value) {
- if (value == null)
- return new BigDecimal(0);
- else
- return new BigDecimal(Bytes.toString(value));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
deleted file mode 100644
index ecaaea7..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- * @author yangli9
- *
- */
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
-
- final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
- static {
- implementations.put("decimal", BigDecimalSerializer.class);
- implementations.put("double", DoubleSerializer.class);
- implementations.put("float", DoubleSerializer.class);
- implementations.put("bigint", LongSerializer.class);
- implementations.put("long", LongSerializer.class);
- implementations.put("integer", LongSerializer.class);
- implementations.put("int", LongSerializer.class);
- }
-
- public static DataTypeSerializer<?> create(String dataType) {
- DataType type = DataType.getInstance(dataType);
- if (type.isHLLC()) {
- return new HLLCSerializer(type.getPrecision());
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + dataType);
-
- try {
- return (DataTypeSerializer<?>) clz.newInstance();
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** peek into buffer and return the length of serialization */
- abstract public int peekLength(ByteBuffer in);
-
- /** convert from String to obj */
- abstract public T valueOf(byte[] value);
-
- public T valueOf(String value) {
- try {
- return valueOf(value.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** convert from obj to string */
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
deleted file mode 100644
index 2874a57..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.DoubleWritable;
-
-/**
- * @author yangli9
- *
- */
-public class DoubleSerializer extends DataTypeSerializer<DoubleWritable> {
-
- // avoid mass object creation
- DoubleWritable current = new DoubleWritable();
-
- @Override
- public void serialize(DoubleWritable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- @Override
- public DoubleWritable deserialize(ByteBuffer in) {
- current.set(in.getDouble());
- return current;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public DoubleWritable valueOf(byte[] value) {
- if (value == null)
- current.set(0d);
- else
- current.set(Double.parseDouble(Bytes.toString(value)));
- return current;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
deleted file mode 100644
index 2303c82..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-
-/**
- * @author yangli9
- *
- */
-public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
-
- HyperLogLogPlusCounter current;
-
- public HLLCSerializer(int p) {
- current = new HyperLogLogPlusCounter(p);
- }
-
- @Override
- public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
- try {
- value.writeRegisters(out);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
- try {
- current.readRegisters(in);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return current;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return current.peekLength(in);
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(byte[] value) {
- current.clear();
- if (value == null)
- current.add("__nUlL__");
- else
- current.add(value);
- return current;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
deleted file mode 100644
index 9dca987..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * @author yangli9
- *
- */
-public class LongSerializer extends DataTypeSerializer<LongWritable> {
-
- // avoid mass object creation
- LongWritable current = new LongWritable();
-
- @Override
- public void serialize(LongWritable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- @Override
- public LongWritable deserialize(ByteBuffer in) {
- current.set(BytesUtil.readVLong(in));
- return current;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- BytesUtil.readVLong(in);
- int len = in.position() - mark;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public LongWritable valueOf(byte[] value) {
- if (value == null)
- current.set(0L);
- else
- current.set(Long.parseLong(Bytes.toString(value)));
- return current;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
index 95a246c..9da0fbe 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
@@ -22,8 +22,8 @@ import java.nio.ByteBuffer;
import java.util.Collection;
import org.apache.hadoop.io.Text;
-
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.serializer.DataTypeSerializer;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
new file mode 100644
index 0000000..0c7b37d
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.serializer;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
+/**
+ * @author yangli9
+ *
+ */
+public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
+
+ @Override
+ public void serialize(BigDecimal value, ByteBuffer out) {
+ byte[] bytes = value.unscaledValue().toByteArray();
+
+ BytesUtil.writeVInt(value.scale(), out);
+ BytesUtil.writeVInt(bytes.length, out);
+ out.put(bytes);
+ }
+
+ @Override
+ public BigDecimal deserialize(ByteBuffer in) {
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+
+ byte[] bytes = new byte[n];
+ in.get(bytes);
+
+ return new BigDecimal(new BigInteger(bytes), scale);
+ }
+
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ @SuppressWarnings("unused")
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+ int len = in.position() - mark + n;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public BigDecimal valueOf(byte[] value) {
+ if (value == null)
+ return new BigDecimal(0);
+ else
+ return new BigDecimal(Bytes.toString(value));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
new file mode 100644
index 0000000..5828178
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.serializer;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ * @author yangli9
+ *
+ */
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+
+ final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
+ static {
+ implementations.put("string", StringSerializer.class);
+ implementations.put("decimal", BigDecimalSerializer.class);
+ implementations.put("double", DoubleSerializer.class);
+ implementations.put("float", DoubleSerializer.class);
+ implementations.put("bigint", LongSerializer.class);
+ implementations.put("long", LongSerializer.class);
+ implementations.put("integer", LongSerializer.class);
+ implementations.put("int", LongSerializer.class);
+ }
+
+ public static DataTypeSerializer<?> create(String dataType) {
+ return create(DataType.getInstance(dataType));
+ }
+
+ public static DataTypeSerializer<?> create(DataType type) {
+ if (type.isHLLC()) {
+ return new HLLCSerializer(type.getPrecision());
+ }
+
+ Class<?> clz = implementations.get(type.getName());
+ if (clz == null)
+ throw new RuntimeException("No MeasureSerializer for type " + type);
+
+ try {
+ return (DataTypeSerializer<?>) clz.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** peek into buffer and return the length of serialization */
+ abstract public int peekLength(ByteBuffer in);
+
+ /** convert from String to obj */
+ abstract public T valueOf(byte[] value);
+
+ public T valueOf(String value) {
+ try {
+ return valueOf(value.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** convert from obj to string */
+ public String toString(T value) {
+ if (value == null)
+ return "NULL";
+ else
+ return value.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
new file mode 100644
index 0000000..455a9d6
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.serializer;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.DoubleWritable;
+
+/**
+ * @author yangli9
+ *
+ */
+public class DoubleSerializer extends DataTypeSerializer<DoubleWritable> {
+
+ // avoid mass object creation
+ DoubleWritable current = new DoubleWritable();
+
+ @Override
+ public void serialize(DoubleWritable value, ByteBuffer out) {
+ out.putDouble(value.get());
+ }
+
+ @Override
+ public DoubleWritable deserialize(ByteBuffer in) {
+ current.set(in.getDouble());
+ return current;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public DoubleWritable valueOf(byte[] value) {
+ if (value == null)
+ current.set(0d);
+ else
+ current.set(Double.parseDouble(Bytes.toString(value)));
+ return current;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/serializer/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/HLLCSerializer.java
new file mode 100644
index 0000000..dafe661
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/HLLCSerializer.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.serializer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+
+/**
+ * @author yangli9
+ *
+ */
+public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
+
+ HyperLogLogPlusCounter current;
+
+ public HLLCSerializer(int p) {
+ current = new HyperLogLogPlusCounter(p);
+ }
+
+ @Override
+ public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
+ try {
+ value.writeRegisters(out);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
+ try {
+ current.readRegisters(in);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return current;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return current.peekLength(in);
+ }
+
+ @Override
+ public HyperLogLogPlusCounter valueOf(byte[] value) {
+ current.clear();
+ if (value == null)
+ current.add("__nUlL__");
+ else
+ current.add(value);
+ return current;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
new file mode 100644
index 0000000..5916e58
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.serializer;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.BytesUtil;
+
+/**
+ * @author yangli9
+ *
+ */
+public class LongSerializer extends DataTypeSerializer<LongWritable> {
+
+ // avoid mass object creation
+ LongWritable current = new LongWritable();
+
+ @Override
+ public void serialize(LongWritable value, ByteBuffer out) {
+ BytesUtil.writeVLong(value.get(), out);
+ }
+
+ @Override
+ public LongWritable deserialize(ByteBuffer in) {
+ current.set(BytesUtil.readVLong(in));
+ return current;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ BytesUtil.readVLong(in);
+ int len = in.position() - mark;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public LongWritable valueOf(byte[] value) {
+ if (value == null)
+ current.set(0L);
+ else
+ current.set(Long.parseLong(Bytes.toString(value)));
+ return current;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
new file mode 100644
index 0000000..56725c0
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
@@ -0,0 +1,30 @@
+package org.apache.kylin.metadata.serializer;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
+public class StringSerializer extends DataTypeSerializer<String> {
+
+ @Override
+ public void serialize(String value, ByteBuffer out) {
+ BytesUtil.writeUTFString(value, out);
+ }
+
+ @Override
+ public String deserialize(ByteBuffer in) {
+ return BytesUtil.readUTFString(in);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return BytesUtil.peekByteArrayLength(in);
+ }
+
+ @Override
+ public String valueOf(byte[] value) {
+ return Bytes.toString(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java
deleted file mode 100644
index b5bc025..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ICodeSystem.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.apache.kylin.metadata.tuple;
-
-import java.nio.ByteBuffer;
-
-/**
- * Decides how constant values are coded and compared.
- *
- * TupleFilter are involved in both query engine and coprocessor. In query engine, the values are strings.
- * In coprocessor, the values are dictionary IDs.
- *
- * The type parameter is java type of code, which should be bytes. However some legacy implementation
- * stores code as String.
- *
- * @author yangli9
- */
-public interface ICodeSystem<T> {
-
- T encode(Object value);
-
- Object decode(T code);
-
- boolean isNull(T code);
-
- int compare(T code1, T code2);
-
- void serialize(T code, ByteBuffer buffer);
-
- T deserialize(ByteBuffer code);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
new file mode 100644
index 0000000..1a14a14
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
@@ -0,0 +1,144 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+
+import com.google.common.collect.Maps;
+
+public class GTAggregateScanner implements IGTScanner {
+
+ final GTInfo info;
+ final BitSet dimensions;
+ final BitSet metrics;
+ final TupleFilter filter;
+ final GTRawScanner rawScanner;
+
+ GTAggregateScanner(GTInfo info, IGTStore store, GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ if (dimensions.intersects(info.colIsMetrics) || metrics.intersects(info.colIsDimension))
+ throw new IllegalStateException();
+
+ this.info = info;
+ this.dimensions = dimensions;
+ this.metrics = metrics;
+ this.filter = filter;
+ this.rawScanner = new GTRawScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics, filter);
+ }
+
+ @Override
+ public void close() throws IOException {
+ rawScanner.close();
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ AggregationCache aggrCache = new AggregationCache();
+ for (GTRecord r : rawScanner) {
+ aggrCache.aggregate(r);
+ }
+ return aggrCache.iterator();
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ class AggregationCache {
+ final SortedMap<GTRecord, MeasureAggregator[]> aggBufMap;
+
+ public AggregationCache() {
+ this.aggBufMap = Maps.newTreeMap();
+ }
+
+ void aggregate(GTRecord r) {
+ r.maskForEqualHashComp = dimensions;
+ MeasureAggregator[] aggrs = aggBufMap.get(r);
+ if (aggrs == null) {
+ aggrs = new MeasureAggregator[metrics.cardinality()];
+ for (int i = 0, col = -1; i < aggrs.length; i++) {
+ col = metrics.nextSetBit(col + 1);
+ aggrs[i] = info.codeSystem.newMetricsAggregator(col);
+ }
+ aggBufMap.put(r.copy(dimensions), aggrs);
+ }
+
+ for (int i = 0, col = -1; i < aggrs.length; i++) {
+ col = metrics.nextSetBit(col + 1);
+ Object metrics = info.codeSystem.decodeColumnValue(col, r.cols[col].wrapAsBuffer());
+ aggrs[i].aggregate(metrics);
+ }
+ }
+
+ public Iterator<GTRecord> iterator() {
+ return new Iterator<GTRecord>() {
+
+ Iterator<Entry<GTRecord, MeasureAggregator[]>> it = aggBufMap.entrySet().iterator();
+ ByteBuffer metricsBuf = ByteBuffer.allocate(info.maxRecordLength);
+ GTRecord oneRecord = new GTRecord(info); // avoid instance creation
+
+ @Override
+ public boolean hasNext() {
+ return it.hasNext();
+ }
+
+ @Override
+ public GTRecord next() {
+ Entry<GTRecord, MeasureAggregator[]> entry = it.next();
+
+ GTRecord dims = entry.getKey();
+ for (int i = dimensions.nextSetBit(0); i >= 0; i = dimensions.nextSetBit(i + 1)) {
+ oneRecord.cols[i].set(dims.cols[i]);
+ }
+
+ metricsBuf.clear();
+ MeasureAggregator[] aggrs = entry.getValue();
+ for (int i = 0, col = -1; i < aggrs.length; i++) {
+ col = metrics.nextSetBit(col + 1);
+ int pos = metricsBuf.position();
+ info.codeSystem.encodeColumnValue(col, aggrs[i].getState(), metricsBuf);
+ oneRecord.cols[col].set(metricsBuf.array(), pos, metricsBuf.position() - pos);
+ }
+
+ return oneRecord;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ public long getSize() {
+ return aggBufMap.size();
+ }
+
+ // ============================================================================
+
+ transient int rowMemBytes;
+ static final int MEMORY_USAGE_CAP = 500 * 1024 * 1024; // 500 MB
+
+ public void checkMemoryUsage() {
+ // about memory calculation,
+ // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/
+ if (rowMemBytes <= 0) {
+ if (aggBufMap.size() > 0) {
+ rowMemBytes = 0;
+ MeasureAggregator[] measureAggregators = aggBufMap.get(aggBufMap.firstKey());
+ for (MeasureAggregator agg : measureAggregators) {
+ rowMemBytes += agg.getMemBytes();
+ }
+ }
+ }
+ int size = aggBufMap.size();
+ int memUsage = (40 + rowMemBytes) * size;
+ if (memUsage > MEMORY_USAGE_CAP) {
+ throw new RuntimeException("Kylin coprocess memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abord coprocessor.");
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
index 0083b86..7e305c5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
@@ -3,21 +3,21 @@ package org.apache.kylin.storage.gridtable;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.kylin.storage.gridtable.GTStore.GTWriter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreWriter;
public class GTBuilder implements Closeable, Flushable {
final private GTInfo info;
- final private GTWriter writer;
+ final private IGTStoreWriter writer;
private GTRowBlock block;
- public GTBuilder(GTInfo info, int shard, GTStore store) {
+ GTBuilder(GTInfo info, int shard, IGTStore store) {
this.info = info;
this.writer = store.rebuild(shard);
- this.block = new GTRowBlock(info);
+ this.block = GTRowBlock.allocate(info);
}
public void write(GTRecord r) throws IOException {
@@ -25,11 +25,8 @@ public class GTBuilder implements Closeable, Flushable {
if (block.isEmpty()) {
makePrimaryKey(r, block.primaryKey);
}
- for (int c = 0; c < info.nColBlocks; c++) {
- ByteBuffer cellBuf = block.cellBlocks[c];
- for (int i = info.colBlockCuts[c], end = info.colBlockCuts[c + 1]; i < end; i++) {
- append(cellBuf, r.cols[i]);
- }
+ for (int i = 0; i < info.colBlocks.length; i++) {
+ r.exportColumnBlock(i, block.cellBlockBuffers[i]);
}
block.nRows++;
@@ -38,24 +35,15 @@ public class GTBuilder implements Closeable, Flushable {
}
}
- private void makePrimaryKey(GTRecord r, ByteBuffer buf) {
- buf.clear();
-
- for (int i : info.primaryKey) {
- append(buf, r.cols[i]);
- }
-
- buf.flip();
- }
-
- private void append(ByteBuffer buf, ByteBuffer data) {
- buf.put(data.array(), data.arrayOffset(), data.limit());
+ private void makePrimaryKey(GTRecord r, ByteArray buf) {
+ r.exportColumns(info.primaryKey, buf);
}
@Override
public void flush() throws IOException {
writer.write(block);
block.clear();
+ block.seqId++;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTCodeSystem.java
new file mode 100644
index 0000000..f9405be
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTCodeSystem.java
@@ -0,0 +1,76 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.metadata.serializer.DataTypeSerializer;
+
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class GTCodeSystem implements IGTCodeSystem {
+
+ final private GTInfo info;
+ final private DataTypeSerializer[] serializers;
+ final private IFilterCodeSystem<ByteArray> filterCS;
+
+ public GTCodeSystem(GTInfo info) {
+ this.info = info;
+
+ this.serializers = new DataTypeSerializer[info.nColumns];
+ for (int i = 0; i < info.nColumns; i++) {
+ this.serializers[i] = DataTypeSerializer.create(info.colTypes[i]);
+ }
+
+ this.filterCS = new IFilterCodeSystem<ByteArray>() {
+ @Override
+ public boolean isNull(ByteArray code) {
+ return (code == null || code.length() == 0);
+ }
+
+ @Override
+ public int compare(ByteArray code1, ByteArray code2) {
+ return code1.compareTo(code2);
+ }
+
+ @Override
+ public void serialize(ByteArray code, ByteBuffer buffer) {
+ BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+ }
+
+ @Override
+ public ByteArray deserialize(ByteBuffer buffer) {
+ return new ByteArray(BytesUtil.readByteArray(buffer));
+ }
+ };
+ }
+
+ @Override
+ public int codeLength(int col, ByteBuffer buf) {
+ return serializers[col].peekLength(buf);
+ }
+
+ @Override
+ public IFilterCodeSystem<ByteArray> getFilterCodeSystem() {
+ return filterCS;
+ }
+
+ // ============================================================================
+
+ @Override
+ public MeasureAggregator<?> newMetricsAggregator(int col) {
+ return MeasureAggregator.create(info.colMetricsAggrFunc[col], info.colTypes[col].getName());
+ }
+
+ @Override
+ public void encodeColumnValue(int col, Object value, ByteBuffer buf) {
+ serializers[col].serialize(value, buf);
+ }
+
+ @Override
+ public Object decodeColumnValue(int col, ByteBuffer buf) {
+ return serializers[col].deserialize(buf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 11a3083..1e2ddd0 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -1,52 +1,190 @@
package org.apache.kylin.storage.gridtable;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+
import org.apache.kylin.metadata.model.DataType;
public class GTInfo {
+ IGTCodeSystem codeSystem;
+ int maxRecordLength = 1024; // column length can vary
+
// column schema
int nColumns;
- DataType[] colType;
- boolean[] colIsMetrics;
- int maxRecordLength; // column length can vary
-
+ DataType[] colTypes;
+ BitSet colIsDimension;
+ BitSet colIsMetrics;
+ BitSet colAll;
+ String[] colMetricsAggrFunc;
+
// grid info
- int[] primaryKey; // columns sorted and unique
- int rowBlockSize; // 0: no row block
- int nColBlocks;
- int[] colBlockCuts; // [i]: the start index of i.th block; [i+1]: the exclusive end index of i.th block
-
+ BitSet primaryKey; // columns sorted and unique
+ BitSet[] colBlocks; // at least one column block
+ int rowBlockSize; // 0: disable row block
+ boolean rowBlockIndexEnabled;
+
// sharding & rowkey
int nShards; // 0: no sharding
- byte[] rowkeyPrefix;
-
+
+ // must create from builder
+ private GTInfo() {
+ }
+
public boolean isShardingEnabled() {
return nShards > 0;
}
-
+
public boolean isRowBlockEnabled() {
return rowBlockSize > 0;
}
-
+
+ public boolean isRowBlockIndexEnabled() {
+ return rowBlockIndexEnabled && isRowBlockEnabled();
+ }
+
void validate() {
- if (nColBlocks != colBlockCuts.length - 1) {
+
+ if (codeSystem == null)
+ throw new IllegalStateException();
+
+ if (primaryKey.cardinality() == 0)
throw new IllegalStateException();
+
+ validateColumns();
+ validateColumnBlocks();
+ }
+
+ private void validateColumns() {
+ colAll = new BitSet();
+ colAll.flip(0, nColumns);
+
+ // dimension and metrics must over all columns
+ if (colIsDimension.intersects(colIsMetrics))
+ throw new IllegalStateException();
+ BitSet all = (BitSet) colAll.clone();
+ all.andNot(colIsDimension);
+ all.andNot(colIsMetrics);
+ if (all.isEmpty() == false)
+ throw new IllegalStateException();
+
+ // column blocks must not overlap
+ for (int i = 0; i < colBlocks.length; i++) {
+ for (int j = i + 1; j < colBlocks.length; j++) {
+ if (colBlocks[i].intersects(colBlocks[j]))
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ private void validateColumnBlocks() {
+ // column block must cover all columns
+ BitSet merge = new BitSet();
+ for (int i = 0; i < colBlocks.length; i++) {
+ merge.or(colBlocks[i]);
+ }
+ if (merge.equals(colAll) == false)
+ throw new IllegalStateException();
+
+ // When row block is disabled, every row is treated as a row block, row PK is same as
+ // row block PK. Thus PK can be removed from column blocks. See <code>GTRowBlock</code>.
+ if (isRowBlockEnabled() == false) {
+ for (int i = 0; i < colBlocks.length; i++) {
+ colBlocks[i].andNot(primaryKey);
+ }
+ }
+
+ // drop empty column block
+ LinkedList<BitSet> tmp = new LinkedList<BitSet>(Arrays.asList(colBlocks));
+ Iterator<BitSet> it = tmp.iterator();
+ while (it.hasNext()) {
+ BitSet cb = it.next();
+ if (cb.isEmpty())
+ it.remove();
+ }
+ colBlocks = (BitSet[]) tmp.toArray(new BitSet[tmp.size()]);
+ }
+
+ public boolean isDimension(int i) {
+ return colIsMetrics.get(i) == false;
+ }
+
+ public boolean isMetrics(int i) {
+ return colIsMetrics.get(i);
+ }
+
+ public static class Builder {
+ final GTInfo info;
+
+ public Builder() {
+ this.info = new GTInfo();
+ }
+
+ /** required */
+ public Builder setCodeSystem(IGTCodeSystem cs) {
+ info.codeSystem = cs;
+ return this;
+ }
+
+ /** required */
+ public Builder setColumns(DataType[] colTypes, BitSet metrics) {
+ info.nColumns = colTypes.length;
+ info.colTypes = colTypes;
+ if (info.colBlocks == null) {
+ BitSet all = new BitSet();
+ all.flip(0, info.nColumns);
+ info.colBlocks = new BitSet[] { all };
+ }
+ return this;
}
- for (int i = 0; i < nColBlocks; i++) {
- int cbStart = colBlockCuts[i], cbEnd = colBlockCuts[i + 1];
- if (cbStart >= cbEnd)
- throw new IllegalStateException();
-
- // when row block disabled, primary key is persisted on rowkey, thus shall not repeat in column block
- if (isRowBlockEnabled() == false) {
- for (int ii : primaryKey) {
- if (ii >= cbStart && ii < cbEnd)
- throw new IllegalStateException();
+ public Builder setMetrics(Map<Integer, String> metricsAggrFunc) {
+ info.colIsDimension = new BitSet();
+ info.colIsMetrics = new BitSet();
+ info.colMetricsAggrFunc = new String[info.nColumns];
+ for (int i = 0; i < info.nColumns; i++) {
+ if (metricsAggrFunc.containsKey(i)) {
+ info.colIsMetrics.set(i);
+ info.colMetricsAggrFunc[i] = metricsAggrFunc.get(i);
+ } else {
+ info.colIsDimension.set(i);
}
}
+ return this;
+ }
+
+ /** required */
+ public Builder setPrimaryKey(BitSet primaryKey) {
+ info.primaryKey = primaryKey;
+ return this;
+ }
+
+ /** optional */
+ public Builder setMaxRecordLength(int len) {
+ info.maxRecordLength = len;
+ return this;
+ }
+
+ /** optional */
+ public Builder enableColumnBlock(BitSet[] columnBlocks) {
+ info.colBlocks = columnBlocks;
+ return this;
}
+ /** optional */
+ public Builder enableRowBlock(int rowBlockSize, boolean enableIndex) {
+ info.rowBlockSize = rowBlockSize;
+ info.rowBlockIndexEnabled = enableIndex;
+ return this;
+ }
+
+ /** optional */
+ public Builder enableSharding(int nShards) {
+ info.nShards = nShards;
+ return this;
+ }
}
-
}
[50/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/af153c42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/af153c42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/af153c42
Branch: refs/heads/streaming-localdict
Commit: af153c42fb07f02d88bd5d207bf1ead64f93f7da
Parents: effd8f8 4e290fb
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 13 21:45:55 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 13 21:45:55 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 72 ++++++-
.../org/apache/kylin/common/util/BytesUtil.java | 21 +++
.../kylin/dict/lookup/LookupBytesTable.java | 2 +-
.../cube/FactDistinctColumnsCombiner.java | 2 +-
.../hadoop/cube/FactDistinctColumnsReducer.java | 2 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 2 +-
.../IIDistinctColumnsCombiner.java | 2 +-
.../invertedindex/IIDistinctColumnsReducer.java | 2 +-
.../kylin/metadata/filter/CaseTupleFilter.java | 7 +-
.../metadata/filter/ColumnTupleFilter.java | 7 +-
.../metadata/filter/CompareTupleFilter.java | 7 +-
.../metadata/filter/ConstantTupleFilter.java | 7 +-
.../metadata/filter/DynamicTupleFilter.java | 7 +-
.../metadata/filter/ExtractTupleFilter.java | 7 +-
.../metadata/filter/IFilterCodeSystem.java | 30 +++
.../metadata/filter/LogicalTupleFilter.java | 13 +-
.../kylin/metadata/filter/StringCodeSystem.java | 15 +-
.../kylin/metadata/filter/TupleFilter.java | 7 +-
.../metadata/filter/TupleFilterSerializer.java | 11 +-
.../metadata/measure/BigDecimalSerializer.java | 75 --------
.../metadata/measure/DataTypeSerializer.java | 83 --------
.../metadata/measure/DoubleSerializer.java | 60 ------
.../kylin/metadata/measure/HLLCSerializer.java | 72 -------
.../kylin/metadata/measure/LongSerializer.java | 67 -------
.../kylin/metadata/measure/MeasureCodec.java | 2 +-
.../serializer/BigDecimalSerializer.java | 75 ++++++++
.../metadata/serializer/DataTypeSerializer.java | 87 +++++++++
.../metadata/serializer/DoubleSerializer.java | 60 ++++++
.../metadata/serializer/HLLCSerializer.java | 72 +++++++
.../metadata/serializer/LongSerializer.java | 67 +++++++
.../metadata/serializer/StringSerializer.java | 30 +++
.../kylin/metadata/tuple/ICodeSystem.java | 30 ---
.../storage/gridtable/GTAggregateScanner.java | 144 ++++++++++++++
.../kylin/storage/gridtable/GTBuilder.java | 32 +---
.../kylin/storage/gridtable/GTCodeSystem.java | 76 ++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 188 ++++++++++++++++---
.../kylin/storage/gridtable/GTRawScanner.java | 169 +++++++++++++++++
.../kylin/storage/gridtable/GTRecord.java | 131 ++++++++++++-
.../kylin/storage/gridtable/GTRowBlock.java | 103 ++++++++--
.../apache/kylin/storage/gridtable/GTStore.java | 35 ----
.../kylin/storage/gridtable/GridTable.java | 25 ++-
.../kylin/storage/gridtable/IGTCodeSystem.java | 25 +++
.../kylin/storage/gridtable/IGTScanner.java | 7 +
.../kylin/storage/gridtable/IGTStore.java | 37 ++++
.../kylin/storage/gridtable/IKVStoreReader.java | 11 --
.../kylin/storage/gridtable/IKVStoreWriter.java | 11 --
.../gridtable/memstore/GTSimpleMemStore.java | 96 ++++++++++
.../hbase/coprocessor/DictCodeSystem.java | 19 +-
.../kylin/storage/filter/FilterBaseTest.java | 3 +-
49 files changed, 1521 insertions(+), 594 deletions(-)
----------------------------------------------------------------------