You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/19 03:05:14 UTC
[01/50] [abbrv] kylin git commit: KYLIN-1726 package rename [Forced
Update!]
Repository: kylin
Updated Branches:
refs/heads/master-cdh5.7 d5efd44c0 -> 5f8b5e816 (forced update)
KYLIN-1726 package rename
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c67fa740
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c67fa740
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c67fa740
Branch: refs/heads/master-cdh5.7
Commit: c67fa740d364d372c7a6424fd160570cc7e890c4
Parents: 5aee022
Author: shaofengshi <sh...@apache.org>
Authored: Sun Oct 9 13:24:59 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 10 13:32:44 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/source/kafka/KafkaMRInput.java | 3 +
.../kylin/source/kafka/MergeOffsetStep.java | 80 -----------
.../kylin/source/kafka/SeekOffsetStep.java | 140 ------------------
.../source/kafka/StringStreamingParser.java | 51 -------
.../apache/kylin/source/kafka/TopicMeta.java | 46 ------
.../kylin/source/kafka/UpdateTimeRangeStep.java | 117 ---------------
.../kylin/source/kafka/job/MergeOffsetStep.java | 80 +++++++++++
.../kylin/source/kafka/job/SeekOffsetStep.java | 141 +++++++++++++++++++
.../source/kafka/job/UpdateTimeRangeStep.java | 117 +++++++++++++++
.../kafka/util/ByteBufferBackedInputStream.java | 6 +-
10 files changed, 344 insertions(+), 437 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 6358ee1..4d1f5c9 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -43,6 +43,9 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.source.kafka.job.MergeOffsetStep;
+import org.apache.kylin.source.kafka.job.SeekOffsetStep;
+import org.apache.kylin.source.kafka.job.UpdateTimeRangeStep;
import javax.annotation.Nullable;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java
deleted file mode 100644
index 18c959a..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.kafka;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class MergeOffsetStep extends AbstractExecutable {
-
- private static final Logger logger = LoggerFactory.getLogger(MergeOffsetStep.class);
- public MergeOffsetStep() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
- final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
-
- List<CubeSegment> mergingSegs = cube.getMergingSegments(segment);
-
- Collections.sort(mergingSegs);
-
- final CubeSegment first = mergingSegs.get(0);
- final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);
-
- segment.setSourceOffsetStart(first.getSourceOffsetStart());
- segment.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
- segment.setSourceOffsetEnd(last.getSourceOffsetEnd());
- segment.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());
-
- long dateRangeStart = CubeManager.minDateRangeStart(mergingSegs);
- long dateRangeEnd = CubeManager.maxDateRangeEnd(mergingSegs);
-
- segment.setDateRangeStart(dateRangeStart);
- segment.setDateRangeEnd(dateRangeEnd);
-
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToUpdateSegs(segment);
- try {
- cubeManager.updateCube(cubeBuilder);
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- } catch (IOException e) {
- logger.error("fail to update cube segment offset", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
deleted file mode 100644
index 151b912..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.kafka;
-
-import org.apache.kylin.source.kafka.util.KafkaClient;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- */
-public class SeekOffsetStep extends AbstractExecutable {
-
- private static final Logger logger = LoggerFactory.getLogger(SeekOffsetStep.class);
-
- public SeekOffsetStep() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
- final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
-
- Map<Integer, Long> startOffsets = segment.getSourcePartitionOffsetStart();
- Map<Integer, Long> endOffsets = segment.getSourcePartitionOffsetEnd();
-
- if (startOffsets.size() > 0 && endOffsets.size() > 0 && startOffsets.size() == endOffsets.size()) {
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "skipped, as the offset is provided.");
- }
-
- final KafkaConfig kafakaConfig = KafkaConfigManager.getInstance(context.getConfig()).getKafkaConfig(cube.getFactTable());
- final String brokers = KafkaClient.getKafkaBrokers(kafakaConfig);
- final String topic = kafakaConfig.getTopic();
- try (final KafkaConsumer consumer = KafkaClient.getKafkaConsumer(brokers, cube.getName(), null)) {
- final List<PartitionInfo> partitionInfos = consumer.partitionsFor(topic);
-
- if (startOffsets.isEmpty()) {
- // user didn't specify start offset, use the biggest offset in existing segments as start
- for (CubeSegment seg : cube.getSegments()) {
- Map<Integer, Long> segEndOffset = seg.getSourcePartitionOffsetEnd();
- for (PartitionInfo partition : partitionInfos) {
- int partitionId = partition.partition();
- if (segEndOffset.containsKey(partitionId)) {
- startOffsets.put(partitionId, Math.max(startOffsets.containsKey(partitionId) ? startOffsets.get(partitionId) : 0, segEndOffset.get(partitionId)));
- }
- }
- }
-
- if (partitionInfos.size() > startOffsets.size()) {
- // has new partition added
- for (int x = startOffsets.size(); x < partitionInfos.size(); x++) {
- long earliest = KafkaClient.getEarliestOffset(consumer, topic, partitionInfos.get(x).partition());
- startOffsets.put(partitionInfos.get(x).partition(), earliest);
- }
- }
-
- logger.info("Get start offset for segment " + segment.getName() + ": " + startOffsets.toString());
- }
-
- if (endOffsets.isEmpty()) {
- // user didn't specify end offset, use latest offset in kafka
- for (PartitionInfo partitionInfo : partitionInfos) {
- long latest = KafkaClient.getLatestOffset(consumer, topic, partitionInfo.partition());
- endOffsets.put(partitionInfo.partition(), latest);
- }
-
- logger.info("Get end offset for segment " + segment.getName() + ": " + endOffsets.toString());
- }
- }
-
- long totalStartOffset = 0, totalEndOffset = 0;
- for (Long v : startOffsets.values()) {
- totalStartOffset += v;
- }
- for (Long v : endOffsets.values()) {
- totalEndOffset += v;
- }
-
- if (totalEndOffset > totalStartOffset) {
- segment.setSourceOffsetStart(totalStartOffset);
- segment.setSourceOffsetEnd(totalEndOffset);
- segment.setSourcePartitionOffsetStart(startOffsets);
- segment.setSourcePartitionOffsetEnd(endOffsets);
- segment.setName(CubeSegment.makeSegmentName(0, 0, totalStartOffset, totalEndOffset));
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToUpdateSegs(segment);
- try {
- cubeManager.updateCube(cubeBuilder);
- } catch (IOException e) {
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed, offset start: " + totalStartOffset + ", offset end: " + totalEndOffset + ", message count: " + (totalEndOffset - totalStartOffset));
- } else {
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToRemoveSegs(segment);
- try {
- cubeManager.updateCube(cubeBuilder);
- } catch (IOException e) {
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
-
- return new ExecuteResult(ExecuteResult.State.DISCARDED, "No new message comes");
- }
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/StringStreamingParser.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StringStreamingParser.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/StringStreamingParser.java
deleted file mode 100644
index f74df83..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StringStreamingParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.kafka;
-
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public final class StringStreamingParser extends StreamingParser {
-
- public static final StringStreamingParser instance = new StringStreamingParser(null, null);
-
- private StringStreamingParser(List<TblColRef> allColumns, Map<String, String> properties) {
- }
-
- @Override
- public StreamingMessage parse(ByteBuffer message) {
- byte[] bytes = new byte[message.limit()];
- message.get(bytes);
- return new StreamingMessage(Lists.newArrayList(new String(bytes).split(",")), 0, 0, Collections.<String, Object> emptyMap());
- }
-
- @Override
- public boolean filter(StreamingMessage streamingMessage) {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
deleted file mode 100644
index a73543e..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.kafka;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * The topic metadata should be invariant, otherwise will cause re-initialization of the Consumer
- *
- */
-public class TopicMeta {
-
- private final String name;
-
- private final List<Integer> partitionIds;
-
- public TopicMeta(String name, List<Integer> partitionIds) {
- this.name = name;
- this.partitionIds = Collections.unmodifiableList(partitionIds);
- }
-
- public String getName() {
- return name;
- }
-
- public List<Integer> getPartitionIds() {
- return partitionIds;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/UpdateTimeRangeStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/UpdateTimeRangeStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/UpdateTimeRangeStep.java
deleted file mode 100644
index 9e902d8..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/UpdateTimeRangeStep.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.kafka;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.FastDateFormat;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class UpdateTimeRangeStep extends AbstractExecutable {
-
- private static final Logger logger = LoggerFactory.getLogger(UpdateTimeRangeStep.class);
-
- public UpdateTimeRangeStep() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
- final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
- final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
- final String outputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- final Path outputFile = new Path(outputPath, partitionCol.getName());
-
- String minValue = null, maxValue = null, currentValue = null;
- FSDataInputStream inputStream = null;
- BufferedReader bufferedReader = null;
- try {
- FileSystem fs = HadoopUtil.getFileSystem(outputPath);
- inputStream = fs.open(outputFile);
- bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
- minValue = currentValue = bufferedReader.readLine();
- while (currentValue != null) {
- maxValue = currentValue;
- currentValue = bufferedReader.readLine();
- }
- } catch (IOException e) {
- logger.error("fail to read file " + outputFile, e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- } finally {
- IOUtils.closeQuietly(bufferedReader);
- IOUtils.closeQuietly(inputStream);
- }
-
- final DataType partitionColType = partitionCol.getType();
- FastDateFormat dateFormat;
- if (partitionColType.isDate()) {
- dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
- } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
- dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- } else if (partitionColType.isStringFamily()) {
- String partitionDateFormat = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
- if (StringUtils.isEmpty(partitionDateFormat)) {
- partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
- }
- dateFormat = DateFormat.getDateFormat(partitionDateFormat);
- } else {
- return new ExecuteResult(ExecuteResult.State.ERROR, "Type " + partitionColType + " is not valid partition column type");
- }
-
- try {
- long startTime = dateFormat.parse(minValue).getTime();
- long endTime = dateFormat.parse(maxValue).getTime();
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- segment.setDateRangeStart(startTime);
- segment.setDateRangeEnd(endTime);
- cubeBuilder.setToUpdateSegs(segment);
- cubeManager.updateCube(cubeBuilder);
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- } catch (Exception e) {
- logger.error("fail to update cube segment offset", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
new file mode 100644
index 0000000..9cadd72
--- /dev/null
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.kafka.job;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class MergeOffsetStep extends AbstractExecutable {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeOffsetStep.class);
+ public MergeOffsetStep() {
+ super();
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+
+ List<CubeSegment> mergingSegs = cube.getMergingSegments(segment);
+
+ Collections.sort(mergingSegs);
+
+ final CubeSegment first = mergingSegs.get(0);
+ final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);
+
+ segment.setSourceOffsetStart(first.getSourceOffsetStart());
+ segment.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
+ segment.setSourceOffsetEnd(last.getSourceOffsetEnd());
+ segment.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());
+
+ long dateRangeStart = CubeManager.minDateRangeStart(mergingSegs);
+ long dateRangeEnd = CubeManager.maxDateRangeEnd(mergingSegs);
+
+ segment.setDateRangeStart(dateRangeStart);
+ segment.setDateRangeEnd(dateRangeEnd);
+
+ CubeUpdate cubeBuilder = new CubeUpdate(cube);
+ cubeBuilder.setToUpdateSegs(segment);
+ try {
+ cubeManager.updateCube(cubeBuilder);
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+ } catch (IOException e) {
+ logger.error("fail to update cube segment offset", e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
new file mode 100644
index 0000000..5751095
--- /dev/null
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.kafka.job;
+
+import org.apache.kylin.source.kafka.KafkaConfigManager;
+import org.apache.kylin.source.kafka.util.KafkaClient;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ */
+public class SeekOffsetStep extends AbstractExecutable {
+
+ private static final Logger logger = LoggerFactory.getLogger(SeekOffsetStep.class);
+
+ public SeekOffsetStep() {
+ super();
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+
+ Map<Integer, Long> startOffsets = segment.getSourcePartitionOffsetStart();
+ Map<Integer, Long> endOffsets = segment.getSourcePartitionOffsetEnd();
+
+ if (startOffsets.size() > 0 && endOffsets.size() > 0 && startOffsets.size() == endOffsets.size()) {
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, "skipped, as the offset is provided.");
+ }
+
+ final KafkaConfig kafakaConfig = KafkaConfigManager.getInstance(context.getConfig()).getKafkaConfig(cube.getFactTable());
+ final String brokers = KafkaClient.getKafkaBrokers(kafakaConfig);
+ final String topic = kafakaConfig.getTopic();
+ try (final KafkaConsumer consumer = KafkaClient.getKafkaConsumer(brokers, cube.getName(), null)) {
+ final List<PartitionInfo> partitionInfos = consumer.partitionsFor(topic);
+
+ if (startOffsets.isEmpty()) {
+ // user didn't specify start offset, use the biggest offset in existing segments as start
+ for (CubeSegment seg : cube.getSegments()) {
+ Map<Integer, Long> segEndOffset = seg.getSourcePartitionOffsetEnd();
+ for (PartitionInfo partition : partitionInfos) {
+ int partitionId = partition.partition();
+ if (segEndOffset.containsKey(partitionId)) {
+ startOffsets.put(partitionId, Math.max(startOffsets.containsKey(partitionId) ? startOffsets.get(partitionId) : 0, segEndOffset.get(partitionId)));
+ }
+ }
+ }
+
+ if (partitionInfos.size() > startOffsets.size()) {
+ // has new partition added
+ for (int x = startOffsets.size(); x < partitionInfos.size(); x++) {
+ long earliest = KafkaClient.getEarliestOffset(consumer, topic, partitionInfos.get(x).partition());
+ startOffsets.put(partitionInfos.get(x).partition(), earliest);
+ }
+ }
+
+ logger.info("Get start offset for segment " + segment.getName() + ": " + startOffsets.toString());
+ }
+
+ if (endOffsets.isEmpty()) {
+ // user didn't specify end offset, use latest offset in kafka
+ for (PartitionInfo partitionInfo : partitionInfos) {
+ long latest = KafkaClient.getLatestOffset(consumer, topic, partitionInfo.partition());
+ endOffsets.put(partitionInfo.partition(), latest);
+ }
+
+ logger.info("Get end offset for segment " + segment.getName() + ": " + endOffsets.toString());
+ }
+ }
+
+ long totalStartOffset = 0, totalEndOffset = 0;
+ for (Long v : startOffsets.values()) {
+ totalStartOffset += v;
+ }
+ for (Long v : endOffsets.values()) {
+ totalEndOffset += v;
+ }
+
+ if (totalEndOffset > totalStartOffset) {
+ segment.setSourceOffsetStart(totalStartOffset);
+ segment.setSourceOffsetEnd(totalEndOffset);
+ segment.setSourcePartitionOffsetStart(startOffsets);
+ segment.setSourcePartitionOffsetEnd(endOffsets);
+ segment.setName(CubeSegment.makeSegmentName(0, 0, totalStartOffset, totalEndOffset));
+ CubeUpdate cubeBuilder = new CubeUpdate(cube);
+ cubeBuilder.setToUpdateSegs(segment);
+ try {
+ cubeManager.updateCube(cubeBuilder);
+ } catch (IOException e) {
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ }
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed, offset start: " + totalStartOffset + ", offset end: " + totalEndOffset + ", message count: " + (totalEndOffset - totalStartOffset));
+ } else {
+ CubeUpdate cubeBuilder = new CubeUpdate(cube);
+ cubeBuilder.setToRemoveSegs(segment);
+ try {
+ cubeManager.updateCube(cubeBuilder);
+ } catch (IOException e) {
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ }
+
+ return new ExecuteResult(ExecuteResult.State.DISCARDED, "No new message comes");
+ }
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/UpdateTimeRangeStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/UpdateTimeRangeStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/UpdateTimeRangeStep.java
new file mode 100644
index 0000000..d19aa63
--- /dev/null
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/UpdateTimeRangeStep.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.kafka.job;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.FastDateFormat;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class UpdateTimeRangeStep extends AbstractExecutable {
+
+ private static final Logger logger = LoggerFactory.getLogger(UpdateTimeRangeStep.class);
+
+ public UpdateTimeRangeStep() {
+ super();
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+ final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
+ final String outputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
+ final Path outputFile = new Path(outputPath, partitionCol.getName());
+
+ String minValue = null, maxValue = null, currentValue = null;
+ FSDataInputStream inputStream = null;
+ BufferedReader bufferedReader = null;
+ try {
+ FileSystem fs = HadoopUtil.getFileSystem(outputPath);
+ inputStream = fs.open(outputFile);
+ bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
+ minValue = currentValue = bufferedReader.readLine();
+ while (currentValue != null) {
+ maxValue = currentValue;
+ currentValue = bufferedReader.readLine();
+ }
+ } catch (IOException e) {
+ logger.error("fail to read file " + outputFile, e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ } finally {
+ IOUtils.closeQuietly(bufferedReader);
+ IOUtils.closeQuietly(inputStream);
+ }
+
+ final DataType partitionColType = partitionCol.getType();
+ FastDateFormat dateFormat;
+ if (partitionColType.isDate()) {
+ dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
+ } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
+ dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ } else if (partitionColType.isStringFamily()) {
+ String partitionDateFormat = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
+ if (StringUtils.isEmpty(partitionDateFormat)) {
+ partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+ }
+ dateFormat = DateFormat.getDateFormat(partitionDateFormat);
+ } else {
+ return new ExecuteResult(ExecuteResult.State.ERROR, "Type " + partitionColType + " is not valid partition column type");
+ }
+
+ try {
+ long startTime = dateFormat.parse(minValue).getTime();
+ long endTime = dateFormat.parse(maxValue).getTime();
+ CubeUpdate cubeBuilder = new CubeUpdate(cube);
+ segment.setDateRangeStart(startTime);
+ segment.setDateRangeEnd(endTime);
+ cubeBuilder.setToUpdateSegs(segment);
+ cubeManager.updateCube(cubeBuilder);
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+ } catch (Exception e) {
+ logger.error("fail to update cube segment offset", e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
index 7a42598..894a144 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.source.kafka.util;
import java.io.IOException;
[06/50] [abbrv] kylin git commit: KYLIN-1923 update context columns
Posted by li...@apache.org.
KYLIN-1923 update context columns
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e23f761e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e23f761e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e23f761e
Branch: refs/heads/master-cdh5.7
Commit: e23f761e7b516e77018f398af6cacab675dbf761
Parents: 7847f51
Author: Cheng Wang <ch...@kyligence.io>
Authored: Sun Oct 9 17:15:51 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Oct 10 15:33:15 2016 +0800
----------------------------------------------------------------------
.../relnode/OLAPToEnumerableConverter.java | 35 +++++++++++++++++---
1 file changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e23f761e/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 95ec617..55c27d3 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -19,6 +19,7 @@
package org.apache.kylin.query.relnode;
import java.util.List;
+import java.util.Set;
import org.apache.calcite.adapter.enumerable.EnumerableRel;
import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
@@ -40,14 +41,18 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.query.routing.QueryRouter;
import org.apache.kylin.query.schema.OLAPTable;
+import com.google.common.collect.Sets;
+
/**
*/
public class OLAPToEnumerableConverter extends ConverterImpl implements EnumerableRel {
@@ -90,7 +95,11 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
if (null != controllerCls && !controllerCls.isEmpty()) {
OLAPContext.IAccessController accessController = (OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
TupleFilter tupleFilter = accessController.check(context.olapAuthen, context.allColumns, context.realization);
- context.filter = and(context.filter, tupleFilter);
+ if (null != tupleFilter) {
+ context.filterColumns.addAll(collectColumns(tupleFilter));
+ context.allColumns.addAll(collectColumns(tupleFilter));
+ context.filter = and(context.filter, tupleFilter);
+ }
}
}
} catch (NoRealizationFoundException e) {
@@ -126,23 +135,41 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
return f2;
if (f2 == null)
return f1;
-
+
if (f1.getOperator() == FilterOperatorEnum.AND) {
f1.addChild(f2);
return f1;
}
-
+
if (f2.getOperator() == FilterOperatorEnum.AND) {
f2.addChild(f1);
return f2;
}
-
+
LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
and.addChild(f1);
and.addChild(f2);
return and;
}
+ private Set<TblColRef> collectColumns(TupleFilter filter) {
+ Set<TblColRef> ret = Sets.newHashSet();
+ collectColumnsRecursively(filter, ret);
+ return ret;
+ }
+
+ private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
+ if (filter == null)
+ return;
+
+ if (filter instanceof ColumnTupleFilter) {
+ collector.add(((ColumnTupleFilter) filter).getColumn());
+ }
+ for (TupleFilter child : filter.getChildren()) {
+ collectColumnsRecursively(child, collector);
+ }
+ }
+
private Result buildHiveResult(EnumerableRelImplementor enumImplementor, Prefer pref, OLAPContext context) {
RelDataType hiveRowType = getRowType();
[29/50] [abbrv] kylin git commit: The SweetAlert at the front page
may out of the page if the content is too long
Posted by li...@apache.org.
The SweetAlert at the front page may out of the page if the content is too long
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70c85a7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70c85a7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70c85a7a
Branch: refs/heads/master-cdh5.7
Commit: 70c85a7a1279ed0246dbc3c61cc47275cd1b31c5
Parents: 81403c1
Author: liguohui <li...@cmss.chinamobile.com>
Authored: Thu Oct 13 15:05:03 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Mon Oct 17 11:20:01 2016 +0800
----------------------------------------------------------------------
webapp/app/less/component.less | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/70c85a7a/webapp/app/less/component.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/component.less b/webapp/app/less/component.less
index e23859d..ed1c9b7 100644
--- a/webapp/app/less/component.less
+++ b/webapp/app/less/component.less
@@ -1169,6 +1169,8 @@ ul.abn-tree li.abn-tree-row a {
.sweet-alert .lead.text-muted {
word-break: break-all;
+ max-height: 300px;
+ overflow: auto;
}
.modal-body.streaming-source .ace_editor {
[09/50] [abbrv] kylin git commit: KYLIN-1726 segment status may have
been updated while job status wasn't committed
Posted by li...@apache.org.
KYLIN-1726 segment status may have been updated while job status wasn't committed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/726b0ec9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/726b0ec9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/726b0ec9
Branch: refs/heads/master-cdh5.7
Commit: 726b0ec93f9f3b1b5da3520be385d6a2047d32c6
Parents: dc969cd
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 10 17:55:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 10 17:55:43 2016 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/726b0ec9/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 269b324..c558c6d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -836,8 +836,9 @@ public class CubeManager implements IRealizationProvider {
if (StringUtils.isBlank(newSegment.getLastBuildJobID()))
throw new IllegalStateException("For cube " + cube + ", segment " + newSegment + " missing LastBuildJobID");
- if (isReady(newSegment) == true)
- throw new IllegalStateException("For cube " + cube + ", segment " + newSegment + " state should be NEW but is READY");
+ if (isReady(newSegment) == true) {
+ logger.warn("For cube " + cube + ", segment " + newSegment + " state should be NEW but is READY");
+ }
List<CubeSegment> tobe = calculateToBeSegments(cube, newSegment);
[15/50] [abbrv] kylin git commit: minor,
controller classes extend from BasicController
Posted by li...@apache.org.
minor, controller classes extend from BasicController
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff851031
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff851031
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff851031
Branch: refs/heads/master-cdh5.7
Commit: ff8510313526134e504651b2d7879c1db8a8388f
Parents: 4be729e
Author: lidongsjtu <li...@apache.org>
Authored: Tue Oct 11 15:36:26 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Oct 11 17:28:13 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller/CubeDescController.java | 2 +-
.../main/java/org/apache/kylin/rest/controller/UserController.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff851031/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 61584a3..4e595cd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -36,7 +36,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
*/
@Controller
@RequestMapping(value = "/cube_desc")
-public class CubeDescController {
+public class CubeDescController extends BasicController {
@Autowired
private CubeService cubeService;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff851031/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
index bccfed4..b0336db 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
@@ -40,7 +40,7 @@ import org.springframework.web.bind.annotation.RequestMethod;
*/
@Controller
@RequestMapping(value = "/user")
-public class UserController {
+public class UserController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(UserController.class);
@Autowired
[47/50] [abbrv] kylin git commit: KYLIN-2105 add QueryId
Posted by li...@apache.org.
KYLIN-2105 add QueryId
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db09f5f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db09f5f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db09f5f9
Branch: refs/heads/master-cdh5.7
Commit: db09f5f9cae5a3d3ff731221cbb1c026da4f4e41
Parents: 6cafc42
Author: gaodayue <ga...@meituan.com>
Authored: Wed Oct 19 08:36:53 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Wed Oct 19 08:36:53 2016 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 6 +
.../apache/kylin/common/util/SetThreadName.java | 49 +++++
.../apache/kylin/rest/service/QueryService.java | 32 ++-
.../kylin/rest/util/QueryIdGenerator.java | 45 +++++
.../kylin/rest/util/QueryIdGeneratorTest.java | 106 ++++++++++
.../src/main/resources/applicationContext.xml | 1 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 7 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../endpoint/generated/CubeVisitProtos.java | 197 +++++++++++++++++--
.../endpoint/protobuf/CubeVisit.proto | 1 +
10 files changed, 420 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f88781a..f7c90aa 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -65,6 +65,10 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
+ public static String getQueryId() {
+ return getString(KEY_QUERY_ID);
+ }
+
public static Pair<Short, Short> getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v == null) {
@@ -92,6 +96,8 @@ public class BackdoorToggles {
_backdoorToggles.remove();
}
+ public final static String KEY_QUERY_ID = "QUERY_ID";
+
/**
* set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java b/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
new file mode 100644
index 0000000..9e5197e
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.Closeable;
+
+/**
+ *
+ * An utility that makes it easy to change current thread name in a try-with-resource block.
+ *
+ * <p> Example:
+ *
+ * <pre>
+ * // old thread name
+ * try (SetThreadName ignored = new SetThreadName(format, arg1, arg2)) {
+ * // new thread name
+ * }
+ * // old thread name
+ * </pre>
+ */
+public class SetThreadName implements Closeable {
+ private final String originThreadName;
+
+ public SetThreadName(String format, Object... args) {
+ originThreadName = Thread.currentThread().getName();
+ Thread.currentThread().setName(String.format(format, args) + "-" + Thread.currentThread().getId());
+ }
+
+ @Override
+ public void close() {
+ Thread.currentThread().setName(originThreadName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index a7ac4b6..2803125 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -55,6 +55,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DBUtils;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -71,6 +72,7 @@ import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.util.QueryIdGenerator;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
@@ -112,10 +114,12 @@ public class QueryService extends BasicService {
private final BadQueryDetector badQueryDetector = new BadQueryDetector();
private final String hbaseUrl;
- private final String tableNameBase;
private final String userTableName;
@Autowired
+ private QueryIdGenerator queryIdGenerator;
+
+ @Autowired
private CacheManager cacheManager;
@Autowired
@@ -130,7 +134,7 @@ public class QueryService extends BasicService {
String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
// split TABLE@HBASE_URL
int cut = metadataUrl.indexOf('@');
- tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
+ String tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
userTableName = tableNameBase + USER_TABLE_NAME;
@@ -264,6 +268,7 @@ public class QueryService extends BasicService {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
+ stringBuilder.append("Query Id: ").append(BackdoorToggles.getQueryId()).append(newLine);
stringBuilder.append("SQL: ").append(request.getSql()).append(newLine);
stringBuilder.append("User: ").append(user).append(newLine);
stringBuilder.append("Success: ").append((null == response.getExceptionMessage())).append(newLine);
@@ -314,20 +319,27 @@ public class QueryService extends BasicService {
}
public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
- try {
- BackdoorToggles.setToggles(sqlRequest.getBackdoorToggles());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ String serverMode = kylinConfig.getServerMode();
+ if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
+ throw new InternalErrorException("Query is not allowed in " + serverMode + " mode.");
+ }
+
+ final String queryId = queryIdGenerator.nextId(sqlRequest.getProject());
+ Map<String, String> toggles = new HashMap<>();
+ toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
+ if (sqlRequest.getBackdoorToggles() != null) {
+ toggles.putAll(sqlRequest.getBackdoorToggles());
+ }
+ BackdoorToggles.setToggles(toggles);
+
+ try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
String sql = sqlRequest.getSql();
String project = sqlRequest.getProject();
logger.info("Using project: " + project);
logger.info("The original query: " + sql);
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- String serverMode = kylinConfig.getServerMode();
- if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
- throw new InternalErrorException("Query is not allowed in " + serverMode + " mode.");
- }
-
if (!sql.toLowerCase().contains("select")) {
logger.debug("Directly return exception as not supported");
throw new InternalErrorException("Not Supported SQL.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
new file mode 100644
index 0000000..d9913e4
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.util;
+
+import org.apache.commons.lang3.time.FastDateFormat;
+
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.ThreadSafe;
+import java.util.concurrent.ThreadLocalRandom;
+
+@ThreadSafe
+public class QueryIdGenerator {
+ private static final char[] base26 = "abcdefghijklmnopqrstuvwxyz".toCharArray();
+ private static final FastDateFormat dateFormat = FastDateFormat.getInstance("yyyyMMdd_HHmmss");
+
+ /**
+ * @param project name of the project
+ * @return the next query id. We try to generate unique id as much as possible, but don't guarantee it.
+ */
+ @Nonnull
+ public String nextId(final String project) {
+ char[] postfix = new char[6];
+ for (int i = 0; i < postfix.length; i++) {
+ postfix[i] = base26[ThreadLocalRandom.current().nextInt(base26.length)];
+ }
+
+ return String.format("%s_%s_%s", dateFormat.format(System.currentTimeMillis()), project, new String(postfix));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java
new file mode 100644
index 0000000..1085206
--- /dev/null
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+public class QueryIdGeneratorTest {
+
+ @Test
+ public void testIdFormat() {
+ QueryIdGenerator generator = new QueryIdGenerator();
+ for (int i = 0; i < 100; i++) {
+ String queryId = generator.nextId("project");
+ Assert.assertTrue(queryId.contains("project"));
+ }
+ }
+
+ @Test
+ public void testIdUniqueness() {
+ QueryIdGenerator generator = new QueryIdGenerator();
+ Set<String> idSet = new HashSet<>();
+
+ for (int i = 0; i < 1000; i++) {
+ idSet.add(generator.nextId("test"));
+ }
+
+ Assert.assertEquals(1000, idSet.size());
+ }
+
+ @Test
+ public void testSingleThreadThroughput() {
+ int N = 1_000_000;
+ long millis = new GenIdTask(new QueryIdGenerator(), N).call();
+
+ // ops / second
+ double throughput = (N * 1000.0) / millis;
+ System.out.format("QueryIdGenerator single thread throughput: %d ops/second\n", (int) throughput);
+ }
+
+ @Test
+ public void testMultiThreadsThroughput() throws ExecutionException, InterruptedException {
+ QueryIdGenerator generator = new QueryIdGenerator();
+ int N = 1_000_000;
+
+ final int numThreads = 4;
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+ Future[] futures = new Future[numThreads];
+
+ for (int i = 0; i < numThreads; i++) {
+ futures[i] = pool.submit(new GenIdTask(generator, N));
+ }
+
+ long sumMillis = 0;
+ for (int i = 0; i < numThreads; i++) {
+ sumMillis += (long) futures[i].get();
+ }
+ pool.shutdown();
+
+ double avgThroughputPerThread = (N * 1000.0) / (sumMillis / (double) numThreads);
+ System.out.format("QueryIdGenerator multi threads throughput: %d ops/second\n", (int) avgThroughputPerThread);
+ }
+
+ private static class GenIdTask implements Callable<Long> {
+ private final QueryIdGenerator generator;
+ private final int N;
+
+ GenIdTask(QueryIdGenerator generator, int N) {
+ this.generator = generator;
+ this.N = N;
+ }
+
+ @Override
+ public Long call() {
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < N; i++) {
+ generator.nextId("test");
+ }
+ return System.currentTimeMillis() - start;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/server/src/main/resources/applicationContext.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/applicationContext.xml b/server/src/main/resources/applicationContext.xml
index 081dc53..c8a4e54 100644
--- a/server/src/main/resources/applicationContext.xml
+++ b/server/src/main/resources/applicationContext.xml
@@ -39,6 +39,7 @@
<aop:aspectj-autoproxy/>
<bean class="org.apache.kylin.rest.init.InitialTaskManager" />
+ <bean class="org.apache.kylin.rest.util.QueryIdGenerator" />
<context:component-scan base-package="org.apache.kylin.rest"/>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index f285153..4f538ae 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -154,13 +155,17 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
builder.setKylinProperties(kylinConfig.getConfigAsString());
+ final String queryId = BackdoorToggles.getQueryId();
+ if (queryId != null) {
+ builder.setQueryId(queryId);
+ }
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
@Override
public void run() {
- final String logHeader = "<sub-thread for GTScanRequest " + Integer.toHexString(System.identityHashCode(scanRequest)) + "> ";
+ final String logHeader = String.format("<sub-thread for Query %s GTScanRequest %s>", queryId, Integer.toHexString(System.identityHashCode(scanRequest)));
final boolean[] abnormalFinish = new boolean[1];
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 3d7b15e..506778c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -46,6 +46,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanExceedThresholdException;
@@ -174,7 +175,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
byte[] allRows;
String debugGitTag = "";
- try {
+ String queryId = request.hasQueryId() ? request.getQueryId() : "UnknownId";
+ try (SetThreadName ignored = new SetThreadName("Kylin Query-%s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
region = env.getRegion();
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index b0688b7..628d453 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -92,6 +92,21 @@ public final class CubeVisitProtos {
*/
com.google.protobuf.ByteString
getKylinPropertiesBytes();
+
+ // optional string queryId = 6;
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ boolean hasQueryId();
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ java.lang.String getQueryId();
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ com.google.protobuf.ByteString
+ getQueryIdBytes();
}
/**
* Protobuf type {@code CubeVisitRequest}
@@ -172,6 +187,11 @@ public final class CubeVisitProtos {
kylinProperties_ = input.readBytes();
break;
}
+ case 50: {
+ bitField0_ |= 0x00000010;
+ queryId_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -858,12 +878,56 @@ public final class CubeVisitProtos {
}
}
+ // optional string queryId = 6;
+ public static final int QUERYID_FIELD_NUMBER = 6;
+ private java.lang.Object queryId_;
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public boolean hasQueryId() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public java.lang.String getQueryId() {
+ java.lang.Object ref = queryId_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ queryId_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public com.google.protobuf.ByteString
+ getQueryIdBytes() {
+ java.lang.Object ref = queryId_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ queryId_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
private void initFields() {
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
rowkeyPreambleSize_ = 0;
hbaseColumnsToGT_ = java.util.Collections.emptyList();
kylinProperties_ = "";
+ queryId_ = "";
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -908,6 +972,9 @@ public final class CubeVisitProtos {
if (((bitField0_ & 0x00000008) == 0x00000008)) {
output.writeBytes(5, getKylinPropertiesBytes());
}
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeBytes(6, getQueryIdBytes());
+ }
getUnknownFields().writeTo(output);
}
@@ -937,6 +1004,10 @@ public final class CubeVisitProtos {
size += com.google.protobuf.CodedOutputStream
.computeBytesSize(5, getKylinPropertiesBytes());
}
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(6, getQueryIdBytes());
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -982,6 +1053,11 @@ public final class CubeVisitProtos {
result = result && getKylinProperties()
.equals(other.getKylinProperties());
}
+ result = result && (hasQueryId() == other.hasQueryId());
+ if (hasQueryId()) {
+ result = result && getQueryId()
+ .equals(other.getQueryId());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1015,6 +1091,10 @@ public final class CubeVisitProtos {
hash = (37 * hash) + KYLINPROPERTIES_FIELD_NUMBER;
hash = (53 * hash) + getKylinProperties().hashCode();
}
+ if (hasQueryId()) {
+ hash = (37 * hash) + QUERYID_FIELD_NUMBER;
+ hash = (53 * hash) + getQueryId().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1139,6 +1219,8 @@ public final class CubeVisitProtos {
}
kylinProperties_ = "";
bitField0_ = (bitField0_ & ~0x00000010);
+ queryId_ = "";
+ bitField0_ = (bitField0_ & ~0x00000020);
return this;
}
@@ -1192,6 +1274,10 @@ public final class CubeVisitProtos {
to_bitField0_ |= 0x00000008;
}
result.kylinProperties_ = kylinProperties_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.queryId_ = queryId_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1248,6 +1334,11 @@ public final class CubeVisitProtos {
kylinProperties_ = other.kylinProperties_;
onChanged();
}
+ if (other.hasQueryId()) {
+ bitField0_ |= 0x00000020;
+ queryId_ = other.queryId_;
+ onChanged();
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -1734,6 +1825,80 @@ public final class CubeVisitProtos {
return this;
}
+ // optional string queryId = 6;
+ private java.lang.Object queryId_ = "";
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public boolean hasQueryId() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public java.lang.String getQueryId() {
+ java.lang.Object ref = queryId_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ queryId_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public com.google.protobuf.ByteString
+ getQueryIdBytes() {
+ java.lang.Object ref = queryId_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ queryId_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public Builder setQueryId(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000020;
+ queryId_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public Builder clearQueryId() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ queryId_ = getDefaultInstance().getQueryId();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string queryId = 6;</code>
+ */
+ public Builder setQueryIdBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000020;
+ queryId_ = value;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:CubeVisitRequest)
}
@@ -4088,25 +4253,25 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\302\001\n\020C" +
+ "/endpoint/protobuf/CubeVisit.proto\"\323\001\n\020C" +
"ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
"\n\014hbaseRawScan\030\002 \002(\014\022\032\n\022rowkeyPreambleSi" +
"ze\030\003 \002(\005\0223\n\020hbaseColumnsToGT\030\004 \003(\0132\031.Cub" +
"eVisitRequest.IntList\022\027\n\017kylinProperties" +
- "\030\005 \002(\t\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\321\002\n\021Cube" +
- "VisitResponse\022\026\n\016compressedRows\030\001 \002(\014\022\'\n" +
- "\005stats\030\002 \002(\0132\030.CubeVisitResponse.Stats\032\372",
- "\001\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022\026\n\016se" +
- "rviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCount\030\003 " +
- "\001(\003\022\032\n\022aggregatedRowCount\030\004 \001(\003\022\025\n\rsyste" +
- "mCpuLoad\030\005 \001(\001\022\036\n\026freePhysicalMemorySize" +
- "\030\006 \001(\001\022\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020\n\010hos" +
- "tname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016normalCom" +
- "plete\030\n \001(\0052F\n\020CubeVisitService\0222\n\tvisit" +
- "Cube\022\021.CubeVisitRequest\032\022.CubeVisitRespo" +
- "nseB`\nEorg.apache.kylin.storage.hbase.cu" +
- "be.v2.coprocessor.endpoint.generatedB\017Cu",
- "beVisitProtosH\001\210\001\001\240\001\001"
+ "\030\005 \002(\t\022\017\n\007queryId\030\006 \001(\t\032\027\n\007IntList\022\014\n\004in" +
+ "ts\030\001 \003(\005\"\321\002\n\021CubeVisitResponse\022\026\n\016compre" +
+ "ssedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisi",
+ "tResponse.Stats\032\372\001\n\005Stats\022\030\n\020serviceStar" +
+ "tTime\030\001 \001(\003\022\026\n\016serviceEndTime\030\002 \001(\003\022\027\n\017s" +
+ "cannedRowCount\030\003 \001(\003\022\032\n\022aggregatedRowCou" +
+ "nt\030\004 \001(\003\022\025\n\rsystemCpuLoad\030\005 \001(\001\022\036\n\026freeP" +
+ "hysicalMemorySize\030\006 \001(\001\022\031\n\021freeSwapSpace" +
+ "Size\030\007 \001(\001\022\020\n\010hostname\030\010 \001(\t\022\016\n\006etcMsg\030\t" +
+ " \001(\t\022\026\n\016normalComplete\030\n \001(\0052F\n\020CubeVisi" +
+ "tService\0222\n\tvisitCube\022\021.CubeVisitRequest" +
+ "\032\022.CubeVisitResponseB`\nEorg.apache.kylin" +
+ ".storage.hbase.cube.v2.coprocessor.endpo",
+ "int.generatedB\017CubeVisitProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4118,7 +4283,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", });
+ new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", "QueryId", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index c84f4f3..1a56719 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -35,6 +35,7 @@ message CubeVisitRequest {
required int32 rowkeyPreambleSize = 3;
repeated IntList hbaseColumnsToGT = 4;
required string kylinProperties = 5; // kylin properties
+ optional string queryId = 6;
message IntList {
repeated int32 ints = 1;
}
[12/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and
data
Posted by li...@apache.org.
KYLIN-2076 Improve sample cube and data
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c7e2688
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c7e2688
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c7e2688
Branch: refs/heads/master-cdh5.7
Commit: 0c7e2688e80822cfbdad901cef909884bfa79ee8
Parents: 726b0ec
Author: Cheng Wang <ch...@kyligence.io>
Authored: Mon Oct 10 17:49:33 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Oct 10 18:02:04 2016 +0800
----------------------------------------------------------------------
examples/sample_cube/create_sample_tables.sql | 2 +
.../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 20000 ++++++++---------
.../cube_desc/kylin_sales_cube_desc.json | 18 +-
.../template/model_desc/kylin_sales_model.json | 4 +-
.../template/table/DEFAULT.KYLIN_SALES.json | 10 +-
5 files changed, 10030 insertions(+), 10004 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index 5ed7b9b..8399ee8 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -181,6 +181,8 @@ TRANS_ID bigint
,PRICE decimal(19,4)
,ITEM_COUNT bigint
,SELLER_ID bigint
+,USER_ID string
+,REGION string
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
[10/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and
data
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index 99f8f46..cc1cd76 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -35,6 +35,16 @@
"table" : "DEFAULT.KYLIN_SALES",
"column" : "LSTG_FORMAT_NAME",
"derived" : null
+ }, {
+ "name" : "USER_ID",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "USER_ID",
+ "derived" : null
+ }, {
+ "name" : "REGION",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "REGION",
+ "derived" : null
} ],
"measures" : [ {
"name" : "GMV_SUM",
@@ -145,6 +155,12 @@
"column" : "LSTG_FORMAT_NAME",
"encoding" : "fixed_length:12"
}, {
+ "column" : "USER_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "REGION",
+ "encoding" : "dict"
+ }, {
"column" : "LSTG_SITE_ID",
"encoding" : "dict"
} ]
@@ -180,4 +196,4 @@
"engine_type" : %default_engine_type%,
"storage_type" : %default_storage_type%,
"override_kylin_properties" : { }
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 7ae47c1..13fc356 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -24,7 +24,7 @@
"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
}, {
"table" : "DEFAULT.KYLIN_SALES",
- "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT" ]
+ "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", "REGION" ]
} ],
"metrics": [
"PRICE",
@@ -40,4 +40,4 @@
"partition_date_start" : 1325376000000,
"partition_type" : "APPEND"
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
index 4927343..2bccbeb 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
@@ -38,7 +38,15 @@
"id" : "9",
"name" : "SELLER_ID",
"datatype" : "bigint"
+ }, {
+ "id" : "10",
+ "name" : "USER_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "11",
+ "name" : "REGION",
+ "datatype" : "string"
} ],
"database" : "DEFAULT",
"last_modified" : 0
-}
\ No newline at end of file
+}
[27/50] [abbrv] kylin git commit: KYLIN-2088 Refactor MeasureType to
allow mutliple UDAF defined on a measure type
Posted by li...@apache.org.
KYLIN-2088 Refactor MeasureType to allow mutliple UDAF defined on a measure type
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0018a212
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0018a212
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0018a212
Branch: refs/heads/master-cdh5.7
Commit: 0018a212470bfe1e8edd3603d383da80f2ffd322
Parents: b4c970a
Author: Yang Li <li...@apache.org>
Authored: Sun Oct 16 23:44:49 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Oct 16 23:44:49 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 1 -
.../org/apache/kylin/measure/MeasureType.java | 16 +++--
.../kylin/measure/MeasureTypeFactory.java | 46 +++++++++++---
.../kylin/measure/basic/BasicMeasureType.java | 6 --
.../kylin/measure/bitmap/BitmapMeasureType.java | 43 ++++++++-----
.../dim/DimCountDistinctMeasureType.java | 10 ++-
.../ExtendedColumnMeasureType.java | 6 +-
.../kylin/measure/hllc/HLLCMeasureType.java | 10 ++-
.../kylin/measure/raw/RawMeasureType.java | 6 +-
.../kylin/measure/topn/TopNMeasureType.java | 5 --
.../kylin/metadata/realization/SQLDigest.java | 43 ++++++++-----
.../kylin/engine/mr/steps/CubeReducerTest.java | 5 +-
.../kylin/storage/hbase/ITStorageTest.java | 3 +-
.../kylin/query/relnode/ColumnRowType.java | 7 +++
.../kylin/query/relnode/OLAPAggregateRel.java | 66 +++++++++++---------
.../apache/kylin/query/relnode/OLAPContext.java | 15 +++--
.../kylin/query/schema/OLAPSchemaFactory.java | 13 +++-
.../cube/MeasureTypeOnlyAggrInBaseTest.java | 5 --
18 files changed, 187 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dacd06..4942081 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -766,7 +766,6 @@ abstract public class KylinConfigBase implements Serializable {
public Map<String, String> getUDFs() {
Map<String, String> udfMap = getPropertiesByPrefix("kylin.query.udf.");
- udfMap.put("intersect_count", "org.apache.kylin.measure.bitmap.BitmapIntersectDistinctCountAggFunc");
return udfMap;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index e7312f2..de1b442 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -102,22 +102,20 @@ abstract public class MeasureType<T> {
* Query Rewrite
* ---------------------------------------------------------------------------- */
- // TODO support user defined Calcite aggr function
-
/** Whether or not Calcite rel-tree needs rewrite to do last around of aggregation */
abstract public boolean needRewrite();
- /** Does the rewrite involves an extra field for the pre-calculated */
+ /** Does the rewrite involves an extra field to hold the pre-calculated */
public boolean needRewriteField() {
return true;
}
- /** Returns a Calcite aggregation function implementation class */
- abstract public Class<?> getRewriteCalciteAggrFunctionClass();
-
- /** Some measure may return different class depends on call name, eg. BitmapMeasureType */
- public Class<?> getRewriteCalciteAggrFunctionClass(String callName) {
- return getRewriteCalciteAggrFunctionClass();
+ /**
+ * Returns a map from UDAF to Calcite aggregation function implementation class.
+ * There can be zero or more UDAF defined on a measure type.
+ */
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return null;
}
/* ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 17d841a..c5bd482 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -32,6 +32,7 @@ import org.apache.kylin.measure.raw.RawMeasureType;
import org.apache.kylin.measure.topn.TopNMeasureType;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,8 +87,10 @@ abstract public class MeasureTypeFactory<T> {
// ============================================================================
- private static Map<String, List<MeasureTypeFactory<?>>> factories = Maps.newHashMap();
- private static List<MeasureTypeFactory<?>> defaultFactory = Lists.newArrayListWithCapacity(2);
+ final private static Map<String, List<MeasureTypeFactory<?>>> factories = Maps.newHashMap();
+ final private static Map<String, Class<?>> udafMap = Maps.newHashMap(); // a map from UDAF to Calcite aggregation function implementation class
+ final private static Map<String, MeasureTypeFactory> udafFactories = Maps.newHashMap(); // a map from UDAF to its owner factory
+ final private static List<MeasureTypeFactory<?>> defaultFactory = Lists.newArrayListWithCapacity(2);
static {
init();
@@ -110,7 +113,8 @@ abstract public class MeasureTypeFactory<T> {
logger.info("Checking custom measure types from kylin config");
try {
- for (String customFactory : KylinConfig.getInstanceFromEnv().getCubeCustomMeasureTypes().values()) {
+ Map<String, String> customMeasureTypes = KylinConfig.getInstanceFromEnv().getCubeCustomMeasureTypes();
+ for (String customFactory : customMeasureTypes.values()) {
try {
logger.info("Checking custom measure types from kylin config: " + customFactory);
factoryInsts.add((MeasureTypeFactory<?>) Class.forName(customFactory).newInstance());
@@ -132,9 +136,10 @@ abstract public class MeasureTypeFactory<T> {
throw new IllegalArgumentException("Aggregation data type name '" + dataTypeName + "' must be in lower case");
Class<? extends DataTypeSerializer<?>> serializer = factory.getAggrDataTypeSerializer();
- logger.info("registering " + dataTypeName);
+ logger.info("registering " + funcName + "(" + dataTypeName + "), " + factory.getClass());
DataType.register(dataTypeName);
DataTypeSerializer.register(dataTypeName, serializer);
+ registerUDAF(factory);
List<MeasureTypeFactory<?>> list = factories.get(funcName);
if (list == null)
factories.put(funcName, list = Lists.newArrayListWithCapacity(2));
@@ -144,13 +149,40 @@ abstract public class MeasureTypeFactory<T> {
defaultFactory.add(new BasicMeasureType.Factory());
}
+ private static void registerUDAF(MeasureTypeFactory<?> factory) {
+ MeasureType<?> type = factory.createMeasureType(factory.getAggrFunctionName(), DataType.getType(factory.getAggrDataTypeName()));
+ Map<String, Class<?>> udafs = type.getRewriteCalciteAggrFunctions();
+ if (type.needRewrite() == false || udafs == null)
+ return;
+
+ for (String udaf : udafs.keySet()) {
+ udaf = udaf.toUpperCase();
+ if (udaf.equals(FunctionDesc.FUNC_COUNT_DISTINCT))
+ continue; // skip built-in function
+
+ if (udafFactories.containsKey(udaf))
+ throw new IllegalStateException("UDAF '" + udaf + "' was dup declared by " + udafFactories.get(udaf) + " and " + factory);
+
+ udafFactories.put(udaf, factory);
+ udafMap.put(udaf, udafs.get(udaf));
+ }
+ }
+
+ public static Map<String, Class<?>> getUDAFs() {
+ return udafMap;
+ }
+
+ public static Map<String, MeasureTypeFactory> getUDAFFactories() {
+ return udafFactories;
+ }
+
public static MeasureType<?> create(String funcName, String dataType) {
return create(funcName, DataType.getType(dataType));
}
public static MeasureType<?> createNoRewriteFieldsMeasureType(String funcName, DataType dataType) {
// currently only has DimCountDistinctAgg
- if (funcName.equalsIgnoreCase("COUNT_DISTINCT")) {
+ if (funcName.equalsIgnoreCase(FunctionDesc.FUNC_COUNT_DISTINCT)) {
return new DimCountDistinctMeasureType.DimCountDistinctMeasureTypeFactory().createMeasureType(funcName, dataType);
}
@@ -213,8 +245,8 @@ abstract public class MeasureTypeFactory<T> {
}
@Override
- public Class getRewriteCalciteAggrFunctionClass() {
- throw new UnsupportedOperationException();
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index 4ab4584..ed493a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -143,10 +143,4 @@ public class BasicMeasureType extends MeasureType {
public boolean needRewrite() {
return !isSum();
}
-
- @Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index 2b88e21..8e2b2f7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -33,12 +33,16 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
+
+import com.google.common.collect.ImmutableMap;
/**
* Created by sunyerui on 15/12/10.
*/
public class BitmapMeasureType extends MeasureType<BitmapCounter> {
- public static final String FUNC_COUNT_DISTINCT = "COUNT_DISTINCT";
+ public static final String FUNC_COUNT_DISTINCT = FunctionDesc.FUNC_COUNT_DISTINCT;
public static final String FUNC_INTERSECT_COUNT_DISTINCT = "INTERSECT_COUNT";
public static final String DATATYPE_BITMAP = "bitmap";
@@ -151,30 +155,37 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
}
}
- @Override
- public boolean needRewrite() {
+ // In order to keep compatibility with old version, tinyint/smallint/int column use value directly, without dictionary
+ private boolean needDictionaryColumn(FunctionDesc functionDesc) {
+ DataType dataType = functionDesc.getParameter().getColRefs().get(0).getType();
+ if (dataType.isIntegerFamily() && !dataType.isBigInt()) {
+ return false;
+ }
return true;
}
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return BitmapDistinctCountAggFunc.class;
+ public boolean needRewrite() {
+ return true;
}
+ static final Map<String, Class<?>> UDAF_MAP = ImmutableMap.<String, Class<?>> of(//
+ FUNC_COUNT_DISTINCT, BitmapDistinctCountAggFunc.class, //
+ FUNC_INTERSECT_COUNT_DISTINCT, BitmapIntersectDistinctCountAggFunc.class);
+
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass(String callName) {
- if (callName != null && callName.equalsIgnoreCase(FUNC_INTERSECT_COUNT_DISTINCT)) {
- return BitmapIntersectDistinctCountAggFunc.class;
- }
- return BitmapDistinctCountAggFunc.class;
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return UDAF_MAP;
}
- // In order to keep compatibility with old version, tinyint/smallint/int column use value directly, without dictionary
- private boolean needDictionaryColumn(FunctionDesc functionDesc) {
- DataType dataType = functionDesc.getParameter().getColRefs().get(0).getType();
- if (dataType.isIntegerFamily() && !dataType.isBigInt()) {
- return false;
+ @Override
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (SQLCall call : sqlDigest.aggrSqlCalls) {
+ if (FUNC_INTERSECT_COUNT_DISTINCT.equals(call.function)) {
+ TblColRef col = (TblColRef) call.args.get(1);
+ if (!sqlDigest.groupbyColumns.contains(col))
+ sqlDigest.groupbyColumns.add(col);
+ }
}
- return true;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
index 9fe1075..0b3fd94 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
@@ -19,15 +19,19 @@
package org.apache.kylin.measure.dim;
import java.util.List;
+import java.util.Map;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.realization.SQLDigest;
+import com.google.common.collect.ImmutableMap;
+
/**
* Created by dongli on 4/20/16.
*/
@@ -76,9 +80,11 @@ public class DimCountDistinctMeasureType extends MeasureType<Object> {
return false;
}
+ static final Map<String, Class<?>> UDAF_MAP = ImmutableMap.<String, Class<?>> of(FunctionDesc.FUNC_COUNT_DISTINCT, DimCountDistinctAggFunc.class);
+
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return DimCountDistinctAggFunc.class;
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return UDAF_MAP;
}
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index 796f1f7..c8f01ad 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -227,6 +227,7 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
};
}
+ @SuppressWarnings("serial")
@Override
public MeasureAggregator<ByteArray> newAggregator() {
return new MeasureAggregator<ByteArray>() {
@@ -268,9 +269,4 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
public boolean needRewrite() {
return false;
}
-
- @Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index bd5013e..0e58dca 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -31,9 +31,11 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.collect.ImmutableMap;
+
public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
- public static final String FUNC_COUNT_DISTINCT = "COUNT_DISTINCT";
+ public static final String FUNC_COUNT_DISTINCT = FunctionDesc.FUNC_COUNT_DISTINCT;
public static final String DATATYPE_HLLC = "hllc";
public static class Factory extends MeasureTypeFactory<HyperLogLogPlusCounter> {
@@ -116,9 +118,11 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
return true;
}
+ static final Map<String, Class<?>> UDAF_MAP = ImmutableMap.<String, Class<?>> of(FUNC_COUNT_DISTINCT, HLLDistinctCountAggFunc.class);
+
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return HLLDistinctCountAggFunc.class;
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return UDAF_MAP;
}
public static boolean isCountDistinct(FunctionDesc func) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 50715ec..3a49d31 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -75,6 +75,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
}
}
+ @SuppressWarnings("unused")
private final DataType dataType;
public RawMeasureType(String funcName, DataType dataType) {
@@ -191,11 +192,6 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
}
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return null;
- }
-
- @Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
if (sqlDigest.isRawQuery) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 800ca88..33ab314 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -288,11 +288,6 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return null;
- }
-
- @Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
for (MeasureDesc measureDesc : measureDescs) {
FunctionDesc topnFunc = measureDesc.getFunction();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index d2bba66..4887abb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -18,7 +18,8 @@
package org.apache.kylin.metadata.realization;
-import java.util.Collection;
+import java.util.List;
+import java.util.Set;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -26,6 +27,8 @@ import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.collect.ImmutableList;
+
/**
*/
public class SQLDigest {
@@ -34,29 +37,41 @@ public class SQLDigest {
ASCENDING, DESCENDING
}
+ public static class SQLCall {
+ public final String function;
+ public final List<Object> args;
+
+ public SQLCall(String function, Iterable<Object> args) {
+ this.function = function;
+ this.args = ImmutableList.copyOf(args);
+ }
+ }
+
public String factTable;
public TupleFilter filter;
- public Collection<JoinDesc> joinDescs;
- public Collection<TblColRef> allColumns;
- public Collection<TblColRef> groupbyColumns;
- public Collection<TblColRef> filterColumns;
- public Collection<TblColRef> metricColumns;
- public Collection<FunctionDesc> aggregations;
- public Collection<MeasureDesc> sortMeasures;
- public Collection<OrderEnum> sortOrders;
+ public List<JoinDesc> joinDescs;
+ public Set<TblColRef> allColumns;
+ public List<TblColRef> groupbyColumns;
+ public Set<TblColRef> filterColumns;
+ public Set<TblColRef> metricColumns;
+ public List<FunctionDesc> aggregations; // storage level measure type, on top of which various sql aggr function may apply
+ public List<SQLCall> aggrSqlCalls; // sql level aggregation function call
+ public List<MeasureDesc> sortMeasures;
+ public List<OrderEnum> sortOrders;
public boolean isRawQuery;
- //initialized when org.apache.kylin.query.routing.QueryRouter.selectRealization()
- public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
- Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc, Collection<MeasureDesc> sortMeasures, Collection<OrderEnum> sortOrders) {
+ public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs, Set<TblColRef> allColumns, //
+ List<TblColRef> groupbyColumns, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, //
+ List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<MeasureDesc> sortMeasures, List<OrderEnum> sortOrders) {
this.factTable = factTable;
this.filter = filter;
this.joinDescs = joinDescs;
this.allColumns = allColumns;
this.groupbyColumns = groupbyColumns;
this.filterColumns = filterColumns;
- this.metricColumns = aggregatedColumns;
- this.aggregations = aggregateFunnc;
+ this.metricColumns = metricColumns;
+ this.aggregations = aggregations;
+ this.aggrSqlCalls = aggrSqlCalls;
this.sortMeasures = sortMeasures;
this.sortOrders = sortOrders;
this.isRawQuery = isRawQuery();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 5e687a5..97dd750 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -27,6 +27,7 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
@@ -197,8 +198,8 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
}
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return origMeasureType.getRewriteCalciteAggrFunctionClass();
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return origMeasureType.getRewriteCalciteAggrFunctions();
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index 136342d..4121c02 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
@@ -33,6 +33,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.IStorageQuery;
@@ -144,7 +145,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
int count = 0;
ITupleIterator iterator = null;
try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
while (iterator.hasNext()) {
ITuple tuple = iterator.next();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
index 095b5e2..f2894e7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
@@ -49,6 +49,13 @@ public class ColumnRowType {
return columns.get(index);
}
+ public TblColRef getColumnByIndexNullable(int index) {
+ if (index < 0 || index >= columns.size())
+ return null;
+ else
+ return columns.get(index);
+ }
+
public int getIndexByName(String columnName) {
for (int i = 0; i < columns.size(); i++) {
if (columns.get(i).getName().equals(columnName)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 8ecb808..9c4f287 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -55,14 +55,14 @@ import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.measure.bitmap.BitmapMeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
import org.apache.kylin.query.schema.OLAPTable;
import com.google.common.base.Preconditions;
@@ -80,23 +80,28 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
AGGR_FUNC_MAP.put("$SUM0", "SUM");
AGGR_FUNC_MAP.put("COUNT", "COUNT");
AGGR_FUNC_MAP.put("COUNT_DISTINCT", "COUNT_DISTINCT");
- AGGR_FUNC_MAP.put(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT, "COUNT_DISTINCT");
AGGR_FUNC_MAP.put("MAX", "MAX");
AGGR_FUNC_MAP.put("MIN", "MIN");
- for (String customAggrFunc : KylinConfig.getInstanceFromEnv().getCubeCustomMeasureTypes().keySet()) {
- AGGR_FUNC_MAP.put(customAggrFunc.trim().toUpperCase(), customAggrFunc.trim().toUpperCase());
+ Map<String, MeasureTypeFactory> udafFactories = MeasureTypeFactory.getUDAFFactories();
+ for (String udaf : udafFactories.keySet()) {
+ AGGR_FUNC_MAP.put(udaf, udafFactories.get(udaf).getAggrFunctionName());
}
}
- private static String getFuncName(AggregateCall aggCall) {
- String aggName = aggCall.getAggregation().getName();
+ private static String getSqlFuncName(AggregateCall aggCall) {
+ String sqlName = aggCall.getAggregation().getName();
if (aggCall.isDistinct()) {
- aggName = aggName + "_DISTINCT";
+ sqlName = sqlName + "_DISTINCT";
}
- String funcName = AGGR_FUNC_MAP.get(aggName);
+ return sqlName;
+ }
+
+ private static String getAggrFuncName(AggregateCall aggCall) {
+ String sqlName = getSqlFuncName(aggCall);
+ String funcName = AGGR_FUNC_MAP.get(sqlName);
if (funcName == null) {
- throw new IllegalStateException("Don't suppoprt aggregation " + aggName);
+ throw new IllegalStateException("Don't suppoprt aggregation " + sqlName);
}
return funcName;
}
@@ -151,7 +156,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// only translate the innermost aggregation
if (!this.afterAggregate) {
- translateGroupBy();
+ this.context.groupByColumns.addAll(this.groups);
this.context.aggregations.addAll(this.aggregations);
this.context.afterAggregate = true;
} else {
@@ -226,15 +231,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
Set<TblColRef> columns = inputColumnRowType.getSourceColumnsByIndex(i);
this.groups.addAll(columns);
}
- // Some UDAF may group data by itself, add group key into groups, prevents aggregate at cube storage server side
- for (AggregateCall aggCall : this.rewriteAggCalls) {
- String aggregateName = aggCall.getAggregation().getName();
- if (aggregateName.equalsIgnoreCase(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)) {
- int index = aggCall.getArgList().get(1);
- TblColRef column = inputColumnRowType.getColumnByIndex(index);
- groups.add(column);
- }
- }
}
private void buildAggregations() {
@@ -254,17 +250,13 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
}
FunctionDesc aggFunc = new FunctionDesc();
- String funcName = getFuncName(aggCall);
+ String funcName = getAggrFuncName(aggCall);
aggFunc.setExpression(funcName);
aggFunc.setParameter(parameter);
this.aggregations.add(aggFunc);
}
}
- private void translateGroupBy() {
- context.groupByColumns.addAll(this.groups);
- }
-
@Override
public void implementRewrite(RewriteImplementor implementor) {
// only rewrite the innermost aggregation
@@ -286,6 +278,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
aggCall = rewriteAggregateCall(aggCall, cubeFunc);
}
this.rewriteAggCalls.add(aggCall);
+ this.context.aggrSqlCalls.add(toSqlCall(aggCall));
}
}
@@ -295,6 +288,18 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
+ private SQLCall toSqlCall(AggregateCall aggCall) {
+ ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
+
+ String function = getSqlFuncName(aggCall);
+ List<Object> args = Lists.newArrayList();
+ for (Integer index : aggCall.getArgList()) {
+ TblColRef col = inputColumnRowType.getColumnByIndexNullable(index);
+ args.add(col);
+ }
+ return new SQLCall(function, args);
+ }
+
private void translateAggregation() {
// now the realization is known, replace aggregations with what's defined on MeasureDesc
List<MeasureDesc> measures = this.context.realization.getMeasures();
@@ -371,8 +376,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
private AggregateCall rewriteAggregateCall(AggregateCall aggCall, FunctionDesc func) {
- //if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields,
- // which do not exist at all
+ // if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields, which do not exist at all
if (noPrecaculatedFieldsAvailable() && func.needRewriteField()) {
logger.info(func + "skip rewriteAggregateCall because no pre-aggregated field available");
return aggCall;
@@ -391,16 +395,18 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
// rebuild function
- String callName = aggCall.getAggregation().getName();
+ String callName = getSqlFuncName(aggCall);
RelDataType fieldType = aggCall.getType();
SqlAggFunction newAgg = aggCall.getAggregation();
+ Map<String, Class<?>> udafMap = func.getMeasureType().getRewriteCalciteAggrFunctions();
if (func.isCount()) {
newAgg = SqlStdOperatorTable.SUM0;
- } else if (func.getMeasureType().getRewriteCalciteAggrFunctionClass() != null) {
- newAgg = createCustomAggFunction(callName, fieldType, func.getMeasureType().getRewriteCalciteAggrFunctionClass(callName));
+ } else if (udafMap != null && udafMap.containsKey(callName)) {
+ newAgg = createCustomAggFunction(callName, fieldType, udafMap.get(callName));
}
// rebuild aggregate call
+ @SuppressWarnings("deprecation")
AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, callName);
return newAggCall;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 41a3b4d..cdec665 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -36,6 +37,7 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
@@ -116,11 +118,12 @@ public class OLAPContext {
// cube metadata
public IRealization realization;
- public Collection<TblColRef> allColumns = new HashSet<TblColRef>();
- public Collection<TblColRef> groupByColumns = new ArrayList<TblColRef>();
- public Collection<TblColRef> metricsColumns = new HashSet<TblColRef>();
- public List<FunctionDesc> aggregations = new ArrayList<FunctionDesc>();
- public Collection<TblColRef> filterColumns = new HashSet<TblColRef>();
+ public Set<TblColRef> allColumns = new HashSet<TblColRef>();
+ public List<TblColRef> groupByColumns = new ArrayList<TblColRef>();
+ public Set<TblColRef> metricsColumns = new HashSet<TblColRef>();
+ public List<FunctionDesc> aggregations = new ArrayList<FunctionDesc>(); // storage level measure type, on top of which various sql aggr function may apply
+ public List<SQLCall> aggrSqlCalls = new ArrayList<SQLCall>(); // sql level aggregation function call
+ public Set<TblColRef> filterColumns = new HashSet<TblColRef>();
public TupleFilter filter;
public List<JoinDesc> joins = new LinkedList<JoinDesc>();
private List<MeasureDesc> sortMeasures;
@@ -144,7 +147,7 @@ public class OLAPContext {
public SQLDigest getSQLDigest() {
if (sqlDigest == null)
- sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations, sortMeasures, sortOrders);
+ sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortMeasures, sortOrders);
return sqlDigest;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index e42d9be..93f06dd 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -25,6 +25,7 @@ import java.io.Writer;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.calcite.schema.Schema;
@@ -34,6 +35,7 @@ import org.apache.calcite.util.ConversionUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.DatabaseDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -41,6 +43,8 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
+
/**
*/
public class OLAPSchemaFactory implements SchemaFactory {
@@ -138,9 +142,14 @@ public class OLAPSchemaFactory implements SchemaFactory {
}
private static void createOLAPSchemaFunctions(Writer out) throws IOException {
- out.write(" \"functions\": [\n");
- Map<String, String> udfs = KylinConfig.getInstanceFromEnv().getUDFs();
+ Map<String, String> udfs = Maps.newHashMap();
+ udfs.putAll(KylinConfig.getInstanceFromEnv().getUDFs());
+ for (Entry<String, Class<?>> entry : MeasureTypeFactory.getUDAFs().entrySet()) {
+ udfs.put(entry.getKey(), entry.getValue().getName());
+ }
+
int index = 0;
+ out.write(" \"functions\": [\n");
for (Map.Entry<String, String> udf : udfs.entrySet()) {
String udfName = udf.getKey().trim().toUpperCase();
String udfClassName = udf.getValue().trim();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
index 1353862..e07ea91 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
@@ -137,11 +137,6 @@ public class MeasureTypeOnlyAggrInBaseTest extends LocalFileMetadataTestCase {
}
@Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return null;
- }
-
- @Override
public boolean onlyAggrInBaseCuboid() {
return true;
}
[35/50] [abbrv] kylin git commit: KYLIN-2099 fix sample table/cube
desc
Posted by li...@apache.org.
KYLIN-2099 fix sample table/cube desc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5429006a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5429006a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5429006a
Branch: refs/heads/master-cdh5.7
Commit: 5429006a5976a52a08ef39976dfedfeb164629ea
Parents: 3924672
Author: Li Yang <li...@apache.org>
Authored: Mon Oct 17 16:42:44 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Oct 17 16:45:31 2016 +0800
----------------------------------------------------------------------
examples/sample_cube/create_sample_tables.sql | 16 ++++++++--------
.../template/cube_desc/kylin_sales_cube_desc.json | 2 +-
.../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 4 ++--
3 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index fa5e25b..9fa17d7 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -116,10 +116,10 @@ CAL_DT date COMMENT 'Date, PK'
,YTM_YN_ID tinyint
,YTQ_YN_ID tinyint
,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
+,KYLIN_CAL_DT_CRE_DATE string
+,KYLIN_CAL_DT_CRE_USER string
+,KYLIN_CAL_DT_UPD_DATE string
+,KYLIN_CAL_DT_UPD_USER string
)
COMMENT 'Date Dimension Table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
@@ -137,10 +137,10 @@ LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
,REGN_CATEG string
,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
-,CRE_DATE string
-,UPD_DATE string COMMENT 'Last Updated Date'
-,CRE_USER string
-,UPD_USER string COMMENT 'Last Updated User'
+,KYLIN_GROUPINGS_CRE_DATE string
+,KYLIN_GROUPINGS_UPD_DATE string COMMENT 'Last Updated Date'
+,KYLIN_GROUPINGS_CRE_USER string
+,KYLIN_GROUPINGS_UPD_USER string COMMENT 'Last Updated User'
,META_CATEG_ID decimal
,META_CATEG_NAME string COMMENT 'Level1 Category'
,CATEG_LVL2_ID decimal
http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index cc1cd76..3e9bff4 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -14,7 +14,7 @@
"name" : "CATEGORY",
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
"column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
}, {
"name" : "CATEGORY_HIERARCHY",
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index 8e97309..accba35 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -40,7 +40,7 @@
"datatype" : "string"
}, {
"id" : "10",
- "name" : "UPD_DATE",
+ "name" : "KYLIN_GROUPINGS_UPD_DATE",
"datatype" : "string"
}, {
"id" : "11",
@@ -48,7 +48,7 @@
"datatype" : "string"
}, {
"id" : "12",
- "name" : "UPD_USER",
+ "name" : "KYLIN_GROUPINGS_UPD_USER",
"datatype" : "string"
}, {
"id" : "13",
[41/50] [abbrv] kylin git commit: KYLIN-2099 fix sample model
Posted by li...@apache.org.
KYLIN-2099 fix sample model
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03118bd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03118bd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03118bd4
Branch: refs/heads/master-cdh5.7
Commit: 03118bd445e7cfba14b75751fb9f478fe650b88a
Parents: c13ab1c
Author: Li Yang <li...@apache.org>
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Oct 18 12:37:13 2016 +0800
----------------------------------------------------------------------
examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/03118bd4/examples/sample_cube/template/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
"columns" : ["CAL_DT", "WEEK_BEG_DT"]
}, {
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+ "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
}, {
"table" : "DEFAULT.KYLIN_SALES",
"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", "REGION" ]
[45/50] [abbrv] kylin git commit: KYLIN-1839 code review
Posted by li...@apache.org.
KYLIN-1839 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa1550c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa1550c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa1550c9
Branch: refs/heads/master-cdh5.7
Commit: aa1550c9ecfb1c047c107fa51187a51e958ab189
Parents: 4473d71
Author: Yang Li <li...@apache.org>
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Oct 19 08:19:35 2016 +0800
----------------------------------------------------------------------
.../engine/mr/common/AbstractHadoopJob.java | 44 +++++++++++---------
1 file changed, 24 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa1550c9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index bbb1711..f70e3bb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -80,7 +79,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job name. For example, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID);
-// @Deprecated
+ // @Deprecated
protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create(BatchConstants.ARG_SEGMENT_NAME);
protected static final Option OPTION_SEGMENT_ID = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_ID).hasArg().isRequired(true).withDescription("Cube segment id").create(BatchConstants.ARG_SEGMENT_ID);
protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input path").create(BatchConstants.ARG_INPUT);
@@ -253,10 +252,10 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
if (!StringUtils.isBlank(mrLibDir)) {
- if(kylinDependency.length() > 0) {
- kylinDependency.append(",");
- }
- kylinDependency.append(mrLibDir);
+ if (kylinDependency.length() > 0) {
+ kylinDependency.append(",");
+ }
+ kylinDependency.append(mrLibDir);
}
setJobTmpJarsAndFiles(job, kylinDependency.toString());
@@ -296,7 +295,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
try {
Configuration jobConf = job.getConfiguration();
- FileSystem fs = FileSystem.getLocal(jobConf);
+ FileSystem localfs = FileSystem.getLocal(jobConf);
FileSystem hdfs = FileSystem.get(jobConf);
StringBuilder jarList = new StringBuilder();
@@ -304,20 +303,29 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
for (String fileName : fNameList) {
Path p = new Path(fileName);
- FileSystem current = (fileName.startsWith(HdfsConstants.HDFS_URI_SCHEME) ? hdfs : fs);
- if(!current.exists(p)) {
- logger.warn("The directory '" + fileName + "for kylin dependency does not exist!!!");
+ if (p.isAbsolute() == false) {
+ logger.warn("The directory of kylin dependency '" + fileName + "' is not absolute, skip");
+ continue;
+ }
+ FileSystem fs;
+ if (hdfs.exists(p)) {
+ fs = hdfs;
+ } else if (localfs.exists(p)) {
+ fs = localfs;
+ } else {
+ logger.warn("The directory of kylin dependency '" + fileName + "' does not exist, skip");
continue;
}
- if (current.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, current, fileName);
+
+ if (fs.getFileStatus(p).isDirectory()) {
+ appendTmpDir(job, fs, p);
continue;
}
StringBuilder list = (p.getName().endsWith(".jar")) ? jarList : fileList;
if (list.length() > 0)
list.append(",");
- list.append(current.getFileStatus(p).getPath());
+ list.append(fs.getFileStatus(p).getPath());
}
appendTmpFiles(fileList.toString(), jobConf);
@@ -327,13 +335,10 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
- private void appendTmpDir(Job job, FileSystem fs, String tmpDir) {
- if (StringUtils.isBlank(tmpDir))
- return;
-
+ private void appendTmpDir(Job job, FileSystem fs, Path tmpDir) {
try {
Configuration jobConf = job.getConfiguration();
- FileStatus[] fList = fs.listStatus(new Path(tmpDir));
+ FileStatus[] fList = fs.listStatus(tmpDir);
StringBuilder jarList = new StringBuilder();
StringBuilder fileList = new StringBuilder();
@@ -341,7 +346,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
for (FileStatus file : fList) {
Path p = file.getPath();
if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, fs, p.toString());
+ appendTmpDir(job, fs, p);
continue;
}
@@ -624,4 +629,3 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
-
[36/50] [abbrv] kylin git commit: KYLIN-2091 Add API to init the
start-point (of each parition) for streaming cube
Posted by li...@apache.org.
KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c92f79ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c92f79ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c92f79ad
Branch: refs/heads/master-cdh5.7
Commit: c92f79ad39e562ed32fff1a55eb979f0593ed6e3
Parents: 5429006
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 17 17:40:57 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 17 17:53:56 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 12 ++++++
.../org/apache/kylin/cube/model/CubeDesc.java | 13 ++++++
.../kylin/measure/topn/TopNMeasureType.java | 2 +-
server-base/pom.xml | 7 ++++
.../kylin/rest/controller/CubeController.java | 10 ++---
.../rest/controller/CubeDescController.java | 44 +++++++++++++++++++-
.../kylin/source/kafka/job/SeekOffsetStep.java | 8 +++-
.../kylin/source/kafka/util/KafkaClient.java | 22 +++++++++-
9 files changed, 108 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4942081..73ac788 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -804,7 +804,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getMaxBuildingSegments() {
- return Integer.parseInt(getOptional("kylin.cube.building.segment.max", "2"));
+ return Integer.parseInt(getOptional("kylin.cube.building.segment.max", "10"));
}
public void setMaxBuildingSegments(int maxBuildingSegments) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..eb5b389 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,6 +564,12 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
public void setSourcePartitionOffsetEnd(Map<Integer, Long> sourcePartitionOffsetEnd) {
this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
+ long total = 0;
+ for (Long x : sourcePartitionOffsetEnd.values()) {
+ total += x;
+ }
+
+ this.sourceOffsetEnd = total;
}
public Map<Integer, Long> getSourcePartitionOffsetStart() {
@@ -572,5 +578,11 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
public void setSourcePartitionOffsetStart(Map<Integer, Long> sourcePartitionOffsetStart) {
this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
+ long total = 0;
+ for (Long x : sourcePartitionOffsetStart.values()) {
+ total += x;
+ }
+
+ this.sourceOffsetStart = total;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..bf3724a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -170,6 +170,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap();
+ @JsonProperty("partition_offset_start")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private Map<Integer, Long> partitionOffsetStart = Maps.newHashMap();
+
public boolean isEnableSharding() {
//in the future may extend to other storage that is shard-able
return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID;
@@ -1011,6 +1015,14 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
this.partitionDateEnd = partitionDateEnd;
}
+ public Map<Integer, Long> getPartitionOffsetStart() {
+ return partitionOffsetStart;
+ }
+
+ public void setPartitionOffsetStart(Map<Integer, Long> partitionOffsetStart) {
+ this.partitionOffsetStart = partitionOffsetStart;
+ }
+
/** Get columns that have dictionary */
public Set<TblColRef> getAllColumnsHaveDictionary() {
Set<TblColRef> result = Sets.newLinkedHashSet();
@@ -1119,6 +1131,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
newCubeDesc.setOverrideKylinProps(cubeDesc.getOverrideKylinProps());
newCubeDesc.setConfig((KylinConfigExt) cubeDesc.getConfig());
newCubeDesc.updateRandomUuid();
+ newCubeDesc.setPartitionOffsetStart(cubeDesc.getPartitionOffsetStart());
return newCubeDesc;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index fcf6d5e..2f93b77 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -262,7 +262,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
};
}
- if (digest.aggregations.size() == 0 ) {
+ if (digest.aggregations.size() == 0) {
// directly query the UHC column without sorting
unmatchedDimensions.removeAll(literalCol);
return new CapabilityInfluence() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index 1302051..67013e4 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -122,6 +122,13 @@
<artifactId>aspectjweaver</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${kafka.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
<!-- Test & Env -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index be242c3..eefc452 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -536,22 +536,22 @@ public class CubeController extends BasicController {
/**
* get cube segment holes
*
- * @return true
+ * @return a list of CubeSegment, each representing a hole
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hole", method = { RequestMethod.GET })
+ @RequestMapping(value = "/{cubeName}/holes", method = { RequestMethod.GET })
@ResponseBody
public List<CubeSegment> getHoles(@PathVariable String cubeName) {
return cubeService.getCubeManager().calculateHoles(cubeName);
}
/**
- * get cube segment holes
+ * fill cube segment holes
*
- * @return true
+ * @return a list of JobInstances to fill the holes
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hole", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/holes", method = { RequestMethod.PUT })
@ResponseBody
public List<JobInstance> fillHoles(@PathVariable String cubeName) {
List<JobInstance> jobs = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 4e595cd..b7eaddd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -19,10 +19,14 @@
package org.apache.kylin.rest.controller;
import java.io.IOException;
+import java.util.Map;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.source.kafka.util.KafkaClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
@@ -68,8 +72,8 @@ public class CubeDescController extends BasicController {
* Get detail information of the "Cube ID"
* return CubeDesc instead of CubeDesc[]
*
- * @param cubeDescName
- * Cube ID
+ * @param cubeName
+ * Cube Name
* @return
* @throws IOException
*/
@@ -88,6 +92,42 @@ public class CubeDescController extends BasicController {
}
}
+ /**
+ * Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
+ * source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.
+ * @param cubeName
+ * @return
+ */
+ @RequestMapping(value = "/{cubeName}/initStartOffsets", method = { RequestMethod.PUT })
+ @ResponseBody
+ public GeneralResponse initStartOffsets(@PathVariable String cubeName) {
+ CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
+
+ String msg = "";
+ if (cubeInstance == null) {
+ msg = "Cube '" + cubeName + "' not found.";
+ throw new IllegalArgumentException(msg);
+ }
+ if (cubeInstance.getSourceType() != ISourceAware.ID_STREAMING) {
+ msg = "Cube '" + cubeName + "' is not a Streaming Cube.";
+ throw new IllegalArgumentException(msg);
+ }
+
+ final GeneralResponse response = new GeneralResponse();
+ try {
+ final Map<Integer, Long> startOffsets = KafkaClient.getCurrentOffsets(cubeInstance);
+ CubeDesc desc = cubeInstance.getDescriptor();
+ desc.setPartitionOffsetStart(startOffsets);
+ cubeService.getCubeDescManager().updateCubeDesc(desc);
+ response.setProperty("result", "success");
+ response.setProperty("offsets", startOffsets.toString());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ return response;
+ }
+
public void setCubeService(CubeService cubeService) {
this.cubeService = cubeService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
index 5751095..2a3dbb5 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
@@ -61,6 +61,12 @@ public class SeekOffsetStep extends AbstractExecutable {
return new ExecuteResult(ExecuteResult.State.SUCCEED, "skipped, as the offset is provided.");
}
+ final Map<Integer, Long> cubeDescStart = cube.getDescriptor().getPartitionOffsetStart();
+ if (cube.getSegments().size() == 1 && cubeDescStart != null && cubeDescStart.size() > 0) {
+ logger.info("This is the first segment, and has initiated the start offsets, will use it");
+ startOffsets = cubeDescStart;
+ }
+
final KafkaConfig kafakaConfig = KafkaConfigManager.getInstance(context.getConfig()).getKafkaConfig(cube.getFactTable());
final String brokers = KafkaClient.getKafkaBrokers(kafakaConfig);
final String topic = kafakaConfig.getTopic();
@@ -110,8 +116,6 @@ public class SeekOffsetStep extends AbstractExecutable {
}
if (totalEndOffset > totalStartOffset) {
- segment.setSourceOffsetStart(totalStartOffset);
- segment.setSourceOffsetEnd(totalEndOffset);
segment.setSourcePartitionOffsetStart(startOffsets);
segment.setSourcePartitionOffsetEnd(endOffsets);
segment.setName(CubeSegment.makeSegmentName(0, 0, totalStartOffset, totalEndOffset));
http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
index 640cc53..685af6a 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
@@ -17,14 +17,19 @@
*/
package org.apache.kylin.source.kafka.util;
+import com.google.common.collect.Maps;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.source.kafka.KafkaConfigManager;
import org.apache.kylin.source.kafka.config.BrokerConfig;
import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -44,7 +49,7 @@ public class KafkaClient {
return producer;
}
- private static Properties constructDefaultKafkaProducerProperties(String brokers, Properties properties){
+ private static Properties constructDefaultKafkaProducerProperties(String brokers, Properties properties) {
Properties props = new Properties();
props.put("bootstrap.servers", brokers);
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
@@ -111,5 +116,20 @@ public class KafkaClient {
return consumer.position(topicPartition);
}
+ public static Map<Integer, Long> getCurrentOffsets(final CubeInstance cubeInstance) {
+ final KafkaConfig kafakaConfig = KafkaConfigManager.getInstance(cubeInstance.getConfig()).getKafkaConfig(cubeInstance.getFactTable());
+ final String brokers = KafkaClient.getKafkaBrokers(kafakaConfig);
+ final String topic = kafakaConfig.getTopic();
+
+ Map<Integer, Long> startOffsets = Maps.newHashMap();
+ try (final KafkaConsumer consumer = KafkaClient.getKafkaConsumer(brokers, cubeInstance.getName(), null)) {
+ final List<PartitionInfo> partitionInfos = consumer.partitionsFor(topic);
+ for (PartitionInfo partitionInfo : partitionInfos) {
+ long latest = KafkaClient.getLatestOffset(consumer, topic, partitionInfo.partition());
+ startOffsets.put(partitionInfo.partition(), latest);
+ }
+ }
+ return startOffsets;
+ }
}
[08/50] [abbrv] kylin git commit: Revert "KYLIN-2078 Can't see
generated SQL at Web UI"
Posted by li...@apache.org.
Revert "KYLIN-2078 Can't see generated SQL at Web UI"
This reverts commit 94a9f381548ff6ecfb253e189a3319252f18d16c.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc969cd5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc969cd5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc969cd5
Branch: refs/heads/master-cdh5.7
Commit: dc969cd5cf7c37b1e0df6b8b6c620b9b9817754d
Parents: 94a9f38
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 10 17:54:08 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 10 17:54:08 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller/CubeController.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dc969cd5/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index bb7dc6a..be242c3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -40,6 +40,7 @@ import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -141,7 +142,8 @@ public class CubeController extends BasicController {
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+ CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY);
+ IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
GeneralResponse repsonse = new GeneralResponse();
[22/50] [abbrv] kylin git commit: KYLIN-1967 Skip empty cube segment
Posted by li...@apache.org.
KYLIN-1967 Skip empty cube segment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6698c8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6698c8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6698c8f
Branch: refs/heads/master-cdh5.7
Commit: e6698c8f054ce925adb041a85c21af8e886aa768
Parents: 5974fc0
Author: Li Yang <li...@apache.org>
Authored: Thu Oct 13 18:55:11 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Oct 13 18:55:59 2016 +0800
----------------------------------------------------------------------
.../kylin/storage/gtrecord/CubeSegmentScanner.java | 2 ++
.../kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 11 ++++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6698c8f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 926ba48..6e19c09 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -69,6 +69,8 @@ public class CubeSegmentScanner implements IGTScanner {
CubeScanRangePlanner scanRangePlanner;
try {
scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics, context);
+ } catch (RuntimeException e) {
+ throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6698c8f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 4ba134d..fa23be1 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -133,7 +133,16 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
continue;
}
}
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage());
+ try {
+ scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage());
+ } catch (IllegalArgumentException ex) {
+ // ref KYLIN-1967, real empty segment can trigger dictionary exception -- IllegalArgumentException: Value not exists!
+ if (cubeSeg.getInputRecords() == 0) {
+ logger.warn("cube segment {} input record is 0, skip it still", cubeSeg);
+ continue;
+ }
+ throw ex;
+ }
scanners.add(scanner);
}
[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 52aa7ea..b78e3d7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -86,7 +87,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -130,8 +131,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -229,6 +230,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -322,8 +324,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -445,11 +447,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -469,7 +471,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -500,8 +501,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -535,13 +536,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -558,7 +558,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 295750a..efda4e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -60,7 +60,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -128,9 +128,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -187,10 +186,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index a1193e7..48ab67c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -40,7 +40,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -76,7 +77,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -159,10 +161,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -184,9 +186,9 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static void resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static void resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
logger.info("Unset coprocessor on " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -214,13 +216,13 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
}
- private static List<String> resetCoprocessorOnHTables(HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(Admin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processed = new ArrayList<String>();
for (String tableName : tableNames) {
@@ -331,7 +333,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index a5a85fa..29ca7b2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -26,10 +26,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 937b65f..1daca0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -31,12 +32,15 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -58,30 +62,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -105,7 +110,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(hBaseAdmin);
+ IOUtils.closeQuietly(admin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index e26c8e8..da13fa4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -119,8 +119,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -135,7 +135,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -170,8 +170,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -206,7 +206,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -224,8 +224,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 81b79cb..8410d48 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 2c2f11c..85f5ad8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -55,6 +57,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,7 +77,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -150,22 +154,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 390930a..1196ed6 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -230,15 +231,8 @@ public class AggregateRegionObserverTest {
return nextRaw(results);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -307,6 +301,11 @@ public class AggregateRegionObserverTest {
return 0;
}
+ @Override
+ public int getBatch() {
+ return 0;
+ }
+
/*
* (non-Javadoc)
*
@@ -323,16 +322,9 @@ public class AggregateRegionObserverTest {
return i < input.size();
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
+ public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+ p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
}
}
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
+ Region first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
[21/50] [abbrv] kylin git commit: KYLIN-2012 rename
RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN
Posted by li...@apache.org.
KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5974fc08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5974fc08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5974fc08
Branch: refs/heads/master-cdh5.7
Commit: 5974fc0870be17a8801c55c7496093d42dbb7c4f
Parents: 36cf99e
Author: gaodayue <ga...@meituan.com>
Authored: Thu Oct 13 17:16:02 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Thu Oct 13 17:16:02 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/CubeDescManager.java | 4 ++--
.../src/main/java/org/apache/kylin/cube/CubeInstance.java | 2 +-
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 6 +++---
.../java/org/apache/kylin/cube/CubeManagerCacheTest.java | 4 ++--
.../kylin/metadata/realization/RealizationStatusEnum.java | 2 +-
.../org/apache/kylin/rest/controller/CubeController.java | 2 +-
.../java/org/apache/kylin/rest/service/JobService.java | 2 +-
webapp/app/partials/cubes/cubes.html | 10 +++++-----
8 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index d6364fe..602707f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -158,10 +158,10 @@ public class CubeDescManager {
cubeDescMap.putLocal(ndesc.getName(), ndesc);
Cuboid.reloadCache(name);
- // if related cube is in DESCBROKEN state before, change it back to DISABLED
+ // if related cube is in BROKEN state before, change it back to DISABLED
CubeManager cubeManager = CubeManager.getInstance(config);
for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
- if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+ if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
cubeManager.reloadCubeLocal(cube.getName());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..b8e9394 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -153,7 +153,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
// in a temporary broken state, so that user can edit and fix it. Broken state is often due to
// schema changes at source.
public boolean allowBrokenDescriptor() {
- return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
+ return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.BROKEN) && segments.isEmpty();
}
public String getResourcePath() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index c558c6d..1ebeb60 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -994,12 +994,12 @@ public class CubeManager implements IRealizationProvider {
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
if (!cubeDesc.getError().isEmpty()) {
- cube.setStatus(RealizationStatusEnum.DESCBROKEN);
+ cube.setStatus(RealizationStatusEnum.BROKEN);
logger.warn("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), cubeName);
- } else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+ } else if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
cube.setStatus(RealizationStatusEnum.DISABLED);
- logger.info("cube {} changed from DESCBROKEN to DISABLED", cubeName);
+ logger.info("cube {} changed from BROKEN to DISABLED", cubeName);
}
cube.setConfig((KylinConfigExt) cubeDesc.getConfig());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index 3f3b462..3a2188f 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -65,11 +65,11 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube");
assertEquals(0, createdCube.getSegments().size());
assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
- createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
+ createdCube.setStatus(RealizationStatusEnum.BROKEN);
CubeUpdate cubeBuilder = new CubeUpdate(createdCube);
cubeManager.updateCube(cubeBuilder);
- assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
+ assertEquals(RealizationStatusEnum.BROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
public CubeDescManager getCubeDescManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
index 27e2d57..2f3d5ca 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
@@ -20,6 +20,6 @@ package org.apache.kylin.metadata.realization;
public enum RealizationStatusEnum {
- DISABLED, READY, DESCBROKEN
+ DISABLED, READY, BROKEN
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index be242c3..cae6e0f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -339,7 +339,7 @@ public class CubeController extends BasicController {
if (cube == null) {
throw new BadRequestException("Cannot find cube " + cubeName);
}
- if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+ if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
throw new BadRequestException("Broken cube can't be cloned");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 589f67d..d32ff00 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -201,7 +201,7 @@ public class JobService extends BasicService {
public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException, JobException {
- if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+ if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
throw new BadRequestException("Broken cube " + cube.getName() + " can't be built");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index de9b99b..a667058 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -67,7 +67,7 @@
</td>
<td>
<span class="label"
- ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED', 'label-warning': cube.status=='DESCBROKEN'}">
+ ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED', 'label-warning': cube.status=='BROKEN'}">
{{ cube.status}}
</span>
</td>
@@ -94,13 +94,13 @@
<li ng-if="cube.status!='READY' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))">
<a ng-click="cubeEdit(cube);">Edit</a></li>
<li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li>
- <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li>
- <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li>
- <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startMerge(cube)">Merge</a></li>
+ <li ng-if="cube.status!='BROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li>
+ <li ng-if="cube.status!='BROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li>
+ <li ng-if="cube.status!='BROKEN'"><a ng-click="startMerge(cube)">Merge</a></li>
<li ng-if="cube.status=='READY'"><a ng-click="disable(cube)">Disable</a></li>
<li ng-if="cube.status=='DISABLED'"><a ng-click="enable(cube)">Enable</a></li>
<li ng-if="cube.status=='DISABLED'"><a ng-click="purge(cube)">Purge</a></li>
- <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="cloneCube(cube)">Clone</a></li>
+ <li ng-if="cube.status!='BROKEN'"><a ng-click="cloneCube(cube)">Clone</a></li>
</ul>
</div>
[34/50] [abbrv] kylin git commit: KYLIN-2098 TopN support query UHC
column without sorting by sum value
Posted by li...@apache.org.
KYLIN-2098 TopN support query UHC column without sorting by sum value
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3924672b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3924672b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3924672b
Branch: refs/heads/master-cdh5.7
Commit: 3924672be05305a9374b4ba757500d89af8e125c
Parents: ef0c9e5
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 17 13:51:05 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 17 13:51:53 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/topn/TopNMeasureType.java | 55 +++++++++++++-------
1 file changed, 35 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3924672b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index b67c374..fcf6d5e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -231,34 +231,49 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
// TopN measure can (and only can) provide one numeric measure and one literal dimension
// e.g. select seller, sum(gmv) from ... group by seller order by 2 desc limit 100
- // check digest requires only one measure
- if (digest.aggregations.size() != 1)
- return null;
-
- // the measure function must be SUM
- FunctionDesc onlyFunction = digest.aggregations.iterator().next();
- if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
- return null;
-
List<TblColRef> literalCol = getTopNLiteralColumn(topN.getFunction());
+ for (TblColRef colRef : literalCol) {
+ if (digest.filterColumns.contains(colRef) == true) {
+ // doesn't allow filtering by topn literal column
+ return null;
+ }
+ }
+
if (unmatchedDimensions.containsAll(literalCol) == false)
return null;
if (digest.groupbyColumns.containsAll(literalCol) == false)
return null;
- for (TblColRef colRef : literalCol) {
- if (digest.filterColumns.contains(colRef) == true) {
+ // check digest requires only one measure
+ if (digest.aggregations.size() == 1) {
+
+ // the measure function must be SUM
+ FunctionDesc onlyFunction = digest.aggregations.iterator().next();
+ if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
return null;
- }
+
+ unmatchedDimensions.removeAll(literalCol);
+ unmatchedAggregations.remove(onlyFunction);
+ return new CapabilityInfluence() {
+ @Override
+ public double suggestCostMultiplier() {
+ return 0.3; // make sure TopN get ahead of other matched realizations
+ }
+ };
}
- unmatchedDimensions.removeAll(literalCol);
- unmatchedAggregations.remove(onlyFunction);
- return new CapabilityInfluence() {
- @Override
- public double suggestCostMultiplier() {
- return 0.3; // make sure TopN get ahead of other matched realizations
- }
- };
+
+ if (digest.aggregations.size() == 0 ) {
+ // directly query the UHC column without sorting
+ unmatchedDimensions.removeAll(literalCol);
+ return new CapabilityInfluence() {
+ @Override
+ public double suggestCostMultiplier() {
+ return 2.0; // topn can answer but with a higher cost
+ }
+ };
+ }
+
+ return null;
}
private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) {
[43/50] [abbrv] kylin git commit: minor, clear MassIn unused code
Posted by li...@apache.org.
minor, clear MassIn unused code
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d28835f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d28835f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d28835f6
Branch: refs/heads/master-cdh5.7
Commit: d28835f69b088d8b08ed0d1bb6514116e280fd13
Parents: c84892f
Author: Roger Shi <ro...@kyligence.io>
Authored: Tue Oct 18 15:42:10 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Oct 18 16:29:43 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/UDF/MassInTupleFilter.java | 9 +-
.../apache/kylin/query/ITMassInQueryTest.java | 3 +
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 12 +-
.../coprocessor/endpoint/CubeVisitService.java | 11 --
.../filter/MassInValueProviderFactoryImpl.java | 43 -------
.../cube/v2/filter/MassInValueProviderImpl.java | 117 -------------------
6 files changed, 12 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 66c9f21..3f0546c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -71,7 +71,14 @@ public class MassInTupleFilter extends FunctionTupleFilter {
@Override
public Collection<?> getValues() {
- return null;
+ if (valueProvider == null) {
+ valueProvider = VALUE_PROVIDER_FACTORY.getProvider(filterTableType, filterTableResourceIdentifier, column);
+ }
+ return valueProvider.getMassInValues();
+ }
+
+ public TblColRef getColumn() {
+ return column;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 1831dd9..5835baf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -41,6 +41,7 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Sets;
@@ -92,11 +93,13 @@ public class ITMassInQueryTest extends KylinTestBase {
}
@Test
+ @Ignore
public void testMassInQuery() throws Exception {
compare("src/test/resources/query/sql_massin", null, true);
}
@Test
+ @Ignore
public void testMassInWithDistinctCount() throws Exception {
run("src/test/resources/query/sql_massin_distinct", null, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 5692000..3cefc5f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -32,19 +32,15 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
-import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.cube.v2.filter.MassInValueProviderFactoryImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,12 +87,6 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo) {
super(segment, cuboid, fullGTInfo);
- MassInTupleFilter.VALUE_PROVIDER_FACTORY = new MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() {
- @Override
- public DimensionEncoding getDimEnc(TblColRef col) {
- return fullGTInfo.getCodeSystem().getDimEnc(col.getColumnDesc().getZeroBasedIndex());
- }
- });
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 4790d6e..3d7b15e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -47,7 +47,6 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanExceedThresholdException;
import org.apache.kylin.gridtable.GTScanRequest;
@@ -56,8 +55,6 @@ import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.cube.v2.CellListIterator;
import org.apache.kylin.storage.hbase.cube.v2.CubeHBaseRPC;
@@ -65,7 +62,6 @@ import org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore;
import org.apache.kylin.storage.hbase.cube.v2.RawScan;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
-import org.apache.kylin.storage.hbase.cube.v2.filter.MassInValueProviderFactoryImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -203,13 +199,6 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
appendProfileInfo(sb, "start latency: " + (this.serviceStartTime - scanReq.getStartTime()));
- MassInTupleFilter.VALUE_PROVIDER_FACTORY = new MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() {
- @Override
- public DimensionEncoding getDimEnc(TblColRef col) {
- return scanReq.getInfo().getCodeSystem().getDimEnc(col.getColumnDesc().getZeroBasedIndex());
- }
- });
-
final List<InnerScannerAsIterator> cellListsForeachRawScan = Lists.newArrayList();
for (RawScan hbaseRawScan : hbaseRawScans) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderFactoryImpl.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderFactoryImpl.java
deleted file mode 100644
index fe6276f..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderFactoryImpl.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v2.filter;
-
-import org.apache.kylin.dimension.DimensionEncoding;
-import org.apache.kylin.metadata.filter.UDF.MassInValueProvider;
-import org.apache.kylin.metadata.filter.UDF.MassInValueProviderFactory;
-import org.apache.kylin.metadata.filter.function.Functions;
-import org.apache.kylin.metadata.model.TblColRef;
-
-public class MassInValueProviderFactoryImpl implements MassInValueProviderFactory {
-
- public interface DimEncAware {
- DimensionEncoding getDimEnc(TblColRef col);
- }
-
- private DimEncAware dimEncAware = null;
-
- public MassInValueProviderFactoryImpl(DimEncAware dimEncAware) {
- this.dimEncAware = dimEncAware;
- }
-
- @Override
- public MassInValueProvider getProvider(Functions.FilterTableType filterTableType, String filterResourceIdentifier, TblColRef col) {
- return new MassInValueProviderImpl(filterTableType, filterResourceIdentifier, dimEncAware.getDimEnc(col));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
deleted file mode 100644
index 9b2df45..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v2.filter;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.dimension.DimensionEncoding;
-import org.apache.kylin.metadata.filter.UDF.MassInValueProvider;
-import org.apache.kylin.metadata.filter.function.Functions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.common.collect.Sets;
-
-public class MassInValueProviderImpl implements MassInValueProvider {
- public static final Logger logger = LoggerFactory.getLogger(MassInValueProviderImpl.class);
-
- private final static Cache<String, Pair<Long, Set<ByteArray>>> hdfs_caches = CacheBuilder.newBuilder().maximumSize(3).removalListener(new RemovalListener<Object, Object>() {
- @Override
- public void onRemoval(RemovalNotification<Object, Object> notification) {
- logger.debug(String.valueOf(notification.getCause()));
- }
- }).build();
-
- private Set<ByteArray> ret = Sets.newHashSet();
-
- public MassInValueProviderImpl(Functions.FilterTableType filterTableType, String filterResourceIdentifier, DimensionEncoding encoding) {
-
- if (filterTableType == Functions.FilterTableType.HDFS) {
-
- logger.info("Start to load HDFS filter table from " + filterResourceIdentifier);
- Stopwatch stopwatch = new Stopwatch().start();
-
- FileSystem fileSystem = null;
- try {
- synchronized (hdfs_caches) {
-
- // directly create hbase configuration here due to no KYLIN_CONF definition.
- fileSystem = FileSystem.get(HBaseConfiguration.create());
-
- long modificationTime = fileSystem.getFileStatus(new Path(filterResourceIdentifier)).getModificationTime();
- Pair<Long, Set<ByteArray>> cached = hdfs_caches.getIfPresent(filterResourceIdentifier);
- if (cached != null && cached.getFirst().equals(modificationTime)) {
- ret = cached.getSecond();
- logger.info("Load HDFS from cache using " + stopwatch.elapsedMillis() + " millis");
- return;
- }
-
- InputStream inputStream = fileSystem.open(new Path(filterResourceIdentifier));
- List<String> lines = IOUtils.readLines(inputStream, Charset.defaultCharset());
-
- logger.info("Load HDFS finished after " + stopwatch.elapsedMillis() + " millis");
-
- for (String line : lines) {
- if (StringUtils.isEmpty(line)) {
- continue;
- }
-
- try {
- ByteArray byteArray = ByteArray.allocate(encoding.getLengthOfEncoding());
- encoding.encode(line.getBytes(), line.getBytes().length, byteArray.array(), 0);
- ret.add(byteArray);
- } catch (Exception e) {
- logger.warn("Error when encoding the filter line " + line);
- }
- }
-
- hdfs_caches.put(filterResourceIdentifier, Pair.newPair(modificationTime, ret));
-
- logger.info("Mass In values constructed after " + stopwatch.elapsedMillis() + " millis, containing " + ret.size() + " entries");
- }
-
- } catch (IOException e) {
- throw new RuntimeException("error when loading the mass in values", e);
- }
- } else {
- throw new RuntimeException("HBASE_TABLE FilterTableType Not supported yet");
- }
- }
-
- @Override
- public Set<?> getMassInValues() {
- return ret;
- }
-}
[14/50] [abbrv] kylin git commit: allow cleanupOldStorage() be
overwritted
Posted by li...@apache.org.
allow cleanupOldStorage() be overwritted
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4be729e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4be729e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4be729e0
Branch: refs/heads/master-cdh5.7
Commit: 4be729e0f7616b24cb289ead635b882abc515e1b
Parents: fad9e81
Author: shaofengshi <sh...@apache.org>
Authored: Tue Oct 11 17:21:08 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Oct 11 17:21:08 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/provision/BuildCubeWithStream.java | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4be729e0/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index d2e46a5..2faa8d0 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -301,7 +301,7 @@ public class BuildCubeWithStream {
HBaseMetadataTestCase.staticCleanupTestMetadata();
}
- private static void cleanupOldStorage() throws Exception {
+ protected void cleanupOldStorage() throws Exception {
String[] args = { "--delete", "true" };
StorageCleanupJob cli = new StorageCleanupJob();
cli.execute(args);
@@ -321,10 +321,6 @@ public class BuildCubeWithStream {
} catch (Throwable e) {
logger.error("error", e);
System.exit(1);
- } finally {
- if (buildCubeWithStream != null) {
- buildCubeWithStream.after();
- }
}
}
[16/50] [abbrv] kylin git commit: KYLIN-2083 more RAM estimation test
for MeasureAggregator and GTAggregateScanner
Posted by li...@apache.org.
KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/460536c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/460536c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/460536c0
Branch: refs/heads/master-cdh5.7
Commit: 460536c001575bd09e9f059062efe0d1049d6b31
Parents: ff85103
Author: gaodayue <ga...@meituan.com>
Authored: Wed Sep 28 17:09:06 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Tue Oct 11 20:36:11 2016 +0800
----------------------------------------------------------------------
core-cube/pom.xml | 5 +
.../gridtable/AggregationCacheMemSizeTest.java | 290 +++++++++----------
core-metadata/pom.xml | 5 +
.../measure/AggregatorMemEstimateTest.java | 104 +++++++
pom.xml | 35 ++-
5 files changed, 292 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7c9a549..39bba59 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -69,6 +69,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index fcd434e..00c0bd0 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -17,197 +17,195 @@
package org.apache.kylin.gridtable;
-import java.math.BigDecimal;
-import java.util.Comparator;
-import java.util.Random;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
+import com.google.common.base.Stopwatch;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
import org.apache.kylin.measure.basic.DoubleSumAggregator;
import org.apache.kylin.measure.basic.LongSumAggregator;
+import org.apache.kylin.measure.bitmap.BitmapAggregator;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
+import org.github.jamm.MemoryMeter;
import org.junit.Test;
+import java.math.BigDecimal;
+import java.util.*;
+
public class AggregationCacheMemSizeTest {
+ private static final MemoryMeter meter = new MemoryMeter();
+ private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
+ private static final Random random = new Random();
+
+ // consider bitmaps with variant cardinality
+ static {
+ for (int i = 0; i < bitmaps.length; i++) {
+ bitmaps[i] = new BitmapCounter();
+ }
- public static final int NUM_OF_OBJS = 1000000 / 2;
+ final int totalBits = 1_000_000;
- interface CreateAnObject {
- Object create();
- }
+ // case 0: sparse, low-cardinality bitmap
+ for (int i = 0; i < 100; i++) {
+ bitmaps[0].add(random.nextInt(totalBits));
+ }
- @Test
- public void testHLLCAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- HLLCAggregator aggr = new HLLCAggregator(10);
- aggr.aggregate(new HyperLogLogPlusCounter(10));
- return aggr;
+ // case 1: 20% full bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(100) < 20) {
+ bitmaps[1].add(i);
}
- });
- System.out.println("HLLC: " + est);
- }
+ }
- @Test
- public void testBigDecimalAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return newBigDecimalAggr();
+ // case 2: half full bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(100) < 50) {
+ bitmaps[2].add(i);
}
+ }
- });
- System.out.println("BigDecimal: " + est);
- }
+ // case 3: 80% full bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(100) < 80) {
+ bitmaps[3].add(i);
+ }
+ }
- private BigDecimalSumAggregator newBigDecimalAggr() {
- BigDecimalSumAggregator aggr = new BigDecimalSumAggregator();
- aggr.aggregate(new BigDecimal("12345678901234567890.123456789"));
- return aggr;
+ // case 4: dense, high-cardinality bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(totalBits) < 100) {
+ continue;
+ }
+ bitmaps[4].add(i);
+ }
}
- @Test
- public void testLongAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return newLongAggr();
- }
- });
- System.out.println("Long: " + est);
+ enum Settings {
+ WITHOUT_MEM_HUNGRY, // only test basic aggrs
+ WITH_HLLC, // basic aggrs + hllc
+ WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap
+ WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap
}
- private LongSumAggregator newLongAggr() {
- LongSumAggregator aggr = new LongSumAggregator();
- aggr.aggregate(new LongMutable(10));
- return aggr;
+ private MeasureAggregator<?>[] createNoMemHungryAggrs() {
+ LongSumAggregator longSum = new LongSumAggregator();
+ longSum.aggregate(new LongMutable(10));
+
+ DoubleSumAggregator doubleSum = new DoubleSumAggregator();
+ doubleSum.aggregate(new DoubleMutable(10));
+
+ BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
+ decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789"));
+
+ return new MeasureAggregator[] { longSum, doubleSum, decimalSum };
}
- @Test
- public void testDoubleAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return newDoubleAggr();
- }
- });
- System.out.println("Double: " + est);
+ private HLLCAggregator createHLLCAggr() {
+ HLLCAggregator hllcAggregator = new HLLCAggregator(14);
+ hllcAggregator.aggregate(new HyperLogLogPlusCounter(14));
+ return hllcAggregator;
}
- private DoubleSumAggregator newDoubleAggr() {
- DoubleSumAggregator aggr = new DoubleSumAggregator();
- aggr.aggregate(new DoubleMutable(10));
- return aggr;
+ private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
+ BitmapCounter counter = new BitmapCounter();
+ counter.merge(lowCardinality ? bitmaps[0] : bitmaps[3]);
+
+ BitmapAggregator result = new BitmapAggregator();
+ result.aggregate(counter);
+ return result;
}
- @Test
- public void testByteArraySize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return new byte[10];
- }
- });
- System.out.println("byte[10]: " + est);
+ private MeasureAggregator<?>[] createAggrs(Settings settings) {
+ List<MeasureAggregator<?>> aggregators = new ArrayList<>();
+ aggregators.addAll(Arrays.asList(createNoMemHungryAggrs()));
+
+ switch (settings) {
+ case WITHOUT_MEM_HUNGRY:
+ break;
+ case WITH_HLLC:
+ aggregators.add(createHLLCAggr());
+ break;
+ case WITH_LOW_CARD_BITMAP:
+ aggregators.add(createBitmapAggr(true));
+ break;
+ case WITH_HIGH_CARD_BITMAP:
+ aggregators.add(createBitmapAggr(false));
+ break;
+ }
+
+ return aggregators.toArray(new MeasureAggregator[aggregators.size()]);
}
@Test
- public void testAggregatorArraySize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return new MeasureAggregator[7];
- }
- });
- System.out.println("MeasureAggregator[7]: " + est);
+ public void testEstimateBitmapMemSize() {
+ BitmapAggregator[] bitmapAggrs = new BitmapAggregator[bitmaps.length];
+ for (int i = 0; i < bitmapAggrs.length; i++) {
+ bitmapAggrs[i] = new BitmapAggregator();
+ bitmapAggrs[i].aggregate(bitmaps[i]);
+ }
+
+ System.out.printf("%-15s %-10s %-10s\n", "cardinality", "estimate", "actual");
+ for (BitmapAggregator aggr : bitmapAggrs) {
+ System.out.printf("%-15d %-10d %-10d\n",
+ aggr.getState().getCount(), aggr.getMemBytesEstimate(), meter.measureDeep(aggr));
+ }
}
@Test
- public void testTreeMapSize() throws InterruptedException {
- final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() {
- @Override
- public int compare(byte[] o1, byte[] o2) {
- return Bytes.compareTo(o1, o2);
- }
- });
- final Random rand = new Random();
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- byte[] key = new byte[10];
- rand.nextBytes(key);
- map.put(key, null);
- return null;
- }
- });
- System.out.println("TreeMap entry: " + (est - 20)); // -20 is to exclude byte[10]
+ public void testEstimateMemSize() throws InterruptedException {
+ int scale = Integer.parseInt(System.getProperty("scale", "1"));
+ scale = Math.max(1, Math.min(10, scale));
+
+ testSetting(Settings.WITHOUT_MEM_HUNGRY, scale * 100000);
+ testSetting(Settings.WITH_HLLC, scale * 5000);
+ testSetting(Settings.WITH_LOW_CARD_BITMAP, scale * 10000);
+ testSetting(Settings.WITH_HIGH_CARD_BITMAP, scale * 1000);
}
- @Test
- public void testAggregationCacheSize() throws InterruptedException {
- final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() {
+ private void testSetting(Settings settings, int inputCount) {
+ SortedMap<byte[], Object> map = new TreeMap<>(new Comparator<byte[]>() {
@Override
public int compare(byte[] o1, byte[] o2) {
return Bytes.compareTo(o1, o2);
}
});
- final Random rand = new Random();
-
- long bytesBefore = memLeft();
- byte[] key = null;
- MeasureAggregator<?>[] aggrs = null;
- for (int i = 0; i < NUM_OF_OBJS; i++) {
- key = new byte[10];
- rand.nextBytes(key);
- aggrs = new MeasureAggregator[4];
- aggrs[0] = newBigDecimalAggr();
- aggrs[1] = newLongAggr();
- aggrs[2] = newDoubleAggr();
- aggrs[3] = newDoubleAggr();
- map.put(key, aggrs);
- }
-
- long bytesAfter = memLeft();
-
- long mapActualSize = bytesBefore - bytesAfter;
- long mapExpectSize = GTAggregateScanner.estimateSizeOfAggrCache(key, aggrs, map.size());
- System.out.println("Actual cache size: " + mapActualSize);
- System.out.println("Expect cache size: " + mapExpectSize);
- }
-
- private int estimateObjectSize(CreateAnObject factory) throws InterruptedException {
- Object[] hold = new Object[NUM_OF_OBJS];
- long bytesBefore = memLeft();
- for (int i = 0; i < hold.length; i++) {
- hold[i] = factory.create();
+ final int reportInterval = inputCount / 10;
+ final Stopwatch stopwatch = new Stopwatch();
+ long estimateMillis = 0;
+ long actualMillis = 0;
+
+ System.out.println("Settings: " + settings);
+ System.out.printf("%15s %15s %15s %15s %15s\n",
+ "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)", "Actual(ms)");
+
+ for (int i = 0; i < inputCount; i++) {
+ byte[] key = new byte[10];
+ random.nextBytes(key);
+ MeasureAggregator[] values = createAggrs(settings);
+ map.put(key, values);
+
+ if ((i+1) % reportInterval == 0) {
+ stopwatch.start();
+ long estimateBytes = GTAggregateScanner.estimateSizeOfAggrCache(key, values, map.size());
+ estimateMillis += stopwatch.elapsedMillis();
+ stopwatch.reset();
+
+ stopwatch.start();
+ long actualBytes = meter.measureDeep(map);
+ actualMillis += stopwatch.elapsedMillis();
+ stopwatch.reset();
+
+ System.out.printf("%,15d %,15d %,15d %,15d %,15d\n",
+ map.size(), estimateBytes, actualBytes, estimateMillis, actualMillis);
+ }
}
+ System.out.println("---------------------------------------\n");
- long bytesAfter = memLeft();
- return (int) ((bytesBefore - bytesAfter) / hold.length);
- }
-
- private long memLeft() throws InterruptedException {
- Runtime.getRuntime().gc();
- Thread.sleep(500);
- return getSystemAvailBytes();
- }
-
- private long getSystemAvailBytes() {
- Runtime runtime = Runtime.getRuntime();
- long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process
- long freeMemory = runtime.freeMemory(); // out of the current heap, how much is free
- long maxMemory = runtime.maxMemory(); // Max heap VM can use e.g. Xmx setting
- long usedMemory = totalMemory - freeMemory; // how much of the current heap the VM is using
- long availableMemory = maxMemory - usedMemory; // available memory i.e. Maximum heap size minus the current amount used
- return availableMemory;
+ map = null;
+ System.gc();
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 142dd33..c95f7f0 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -65,6 +65,11 @@
</dependency>
<dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
new file mode 100644
index 0000000..2883923
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -0,0 +1,104 @@
+package org.apache.kylin.measure;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.measure.basic.*;
+import org.apache.kylin.measure.bitmap.BitmapAggregator;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
+import org.apache.kylin.measure.hllc.HLLCAggregator;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.github.jamm.MemoryMeter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
+ private static final MemoryMeter meter = new MemoryMeter();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ staticCreateTestMetadata();
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ cleanAfterClass();
+ }
+
+ private List<? extends MeasureAggregator> basicAggregators() {
+ LongMutable longVal = new LongMutable(1000);
+ LongMinAggregator longMin = new LongMinAggregator();
+ LongMaxAggregator longMax = new LongMaxAggregator();
+ LongSumAggregator longSum = new LongSumAggregator();
+ longMin.aggregate(longVal);
+ longMax.aggregate(longVal);
+ longSum.aggregate(longVal);
+
+ DoubleMutable doubleVal = new DoubleMutable(1.0);
+ DoubleMinAggregator doubleMin = new DoubleMinAggregator();
+ DoubleMaxAggregator doubleMax = new DoubleMaxAggregator();
+ DoubleSumAggregator doubleSum = new DoubleSumAggregator();
+ doubleMin.aggregate(doubleVal);
+ doubleMax.aggregate(doubleVal);
+ doubleSum.aggregate(doubleVal);
+
+ BigDecimalMinAggregator decimalMin = new BigDecimalMinAggregator();
+ BigDecimalMaxAggregator decimalMax = new BigDecimalMaxAggregator();
+ BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
+ BigDecimal decimal = new BigDecimal("12345678901234567890.123456789");
+ decimalMin.aggregate(decimal);
+ decimalMax.aggregate(decimal);
+ decimalSum.aggregate(decimal);
+
+ return Lists.newArrayList(
+ longMin, longMax, longSum,
+ doubleMin, doubleMax, doubleSum,
+ decimalMin, decimalMax, decimalSum
+ );
+ }
+
+ private String getAggregatorName(Class<? extends MeasureAggregator> clazz) {
+ if (!clazz.isAnonymousClass()) {
+ return clazz.getSimpleName();
+ }
+ String[] parts = clazz.getName().split("\\.");
+ return parts[parts.length - 1];
+ }
+
+ @Test
+ public void testAggregatorEstimate() {
+ HLLCAggregator hllcAggregator = new HLLCAggregator(14);
+ hllcAggregator.aggregate(new HyperLogLogPlusCounter(14));
+
+ BitmapAggregator bitmapAggregator = new BitmapAggregator();
+ BitmapCounter bitmapCounter = new BitmapCounter();
+ for (int i = 4000; i <= 100000; i += 2) {
+ bitmapCounter.add(i);
+ }
+ bitmapAggregator.aggregate(bitmapCounter);
+
+ ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType("EXTENDED_COLUMN", DataType.getType("extendedcolumn(100)"));
+ MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator();
+ extendedColumnAggregator.aggregate(new ByteArray(100));
+
+ List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators());
+ aggregators.add(hllcAggregator);
+ aggregators.add(bitmapAggregator);
+ aggregators.add(extendedColumnAggregator);
+
+ System.out.printf("%40s %10s %10s\n", "Class", "Estimate", "Actual");
+ for (MeasureAggregator aggregator : aggregators) {
+ String clzName = getAggregatorName(aggregator.getClass());
+ System.out.printf("%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 72e4069..caa09ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,7 @@
<dbunit.version>2.5.2</dbunit.version>
<h2.version>1.4.192</h2.version>
<jetty.version>9.3.10.v20160621</jetty.version>
+ <jamm.version>0.3.1</jamm.version>
<!-- Commons -->
<commons-lang.version>2.6</commons-lang.version>
@@ -122,6 +123,8 @@
org/apache/kylin/**/tools/**:**/*CLI.java
</sonar.jacoco.excludes>
+ <!-- JVM Args for Testing -->
+ <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine>
</properties>
<licenses>
@@ -635,6 +638,11 @@
<artifactId>kryo-shaded</artifactId>
<version>${kryo.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <version>${jamm.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@@ -869,6 +877,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
+ <version>2.10</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1046,6 +1055,30 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-jamm</id>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <phase>generate-test-resources</phase>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
+ <destFileName>jamm.jar</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
<configuration>
@@ -1067,7 +1100,7 @@
<value>kylin-log4j.properties</value>
</property>
</systemProperties>
- <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine>
+ <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine}</argLine>
</configuration>
</plugin>
[39/50] [abbrv] kylin git commit: KYLIN-2095 Allow cube to override
Hive job configuration by properties
Posted by li...@apache.org.
KYLIN-2095 Allow cube to override Hive job configuration by properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07e81fd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07e81fd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07e81fd0
Branch: refs/heads/master-cdh5.7
Commit: 07e81fd0b744e782d84383e327b1923cfc178d42
Parents: cc2b59f
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 17 22:07:57 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 17 22:07:57 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/source/hive/HiveMRInput.java | 42 ++++++++++++--------
2 files changed, 30 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/07e81fd0/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 73ac788..5a06813 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -250,6 +250,10 @@ abstract public class KylinConfigBase implements Serializable {
return getPropertiesByPrefix("kylin.job.mr.config.override.");
}
+ public Map<String, String> getHiveConfigOverride() {
+ return getPropertiesByPrefix("kylin.hive.config.override.");
+ }
+
public String getKylinSparkJobJarPath() {
final String jobJar = getOptional("kylin.job.jar.spark");
if (StringUtils.isNotEmpty(jobJar)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/07e81fd0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 4ec8d3d..f3fceb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -20,6 +20,8 @@ package org.apache.kylin.source.hive;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
@@ -61,7 +63,6 @@ import com.google.common.collect.Sets;
public class HiveMRInput implements IMRInput {
- private static final String MR_OVERRIDE_JOB_QUEUENAME = "mapreduce.job.queuename";
@Override
public IMRBatchCubingInputSide getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
@@ -157,9 +158,7 @@ public class HiveMRInput implements IMRInput {
hiveInitBuf.append("USE ").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
- if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
- hiveInitBuf.append("SET mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
- }
+ appendHiveOverrideProperties(kylinConfig, hiveInitBuf);
String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
RedistributeFlatHiveTableStep step = new RedistributeFlatHiveTableStep();
@@ -178,10 +177,7 @@ public class HiveMRInput implements IMRInput {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
- if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
- hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
- kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
- }
+ appendHiveOverrideProperties2(kylinConfig, hiveCmdBuilder);
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
hiveCmdBuilder.addStatement("set hive.exec.compress.output=false;\n");
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc, rowCountOutputDir));
@@ -192,6 +188,7 @@ public class HiveMRInput implements IMRInput {
return step;
}
+
public ShellExecutable createLookupHiveViewMaterializationStep(String jobId) {
ShellExecutable step = new ShellExecutable();
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
@@ -211,10 +208,7 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
- if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
- hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
- kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
- }
+ appendHiveOverrideProperties2(kylinConfig, hiveCmdBuilder);
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";";
hiveCmdBuilder.addStatement(useDatabaseHql);
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
@@ -240,9 +234,7 @@ public class HiveMRInput implements IMRInput {
StringBuilder hiveInitBuf = new StringBuilder();
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
- if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
- hiveInitBuf.append("SET mapreduce.job.queuename =").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
- }
+ appendHiveOverrideProperties(kylinConfig, hiveInitBuf);
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";\n";
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId));
@@ -301,7 +293,7 @@ public class HiveMRInput implements IMRInput {
FileSystem fs = FileSystem.get(file.toUri(), HadoopUtil.getCurrentConfiguration());
InputStream in = fs.open(file);
try {
- String content = IOUtils.toString(in);
+ String content = IOUtils.toString(in, Charset.defaultCharset());
return Long.valueOf(content.trim()); // strip the '\n' character
} finally {
@@ -490,4 +482,22 @@ public class HiveMRInput implements IMRInput {
}
}
+
+ private static void appendHiveOverrideProperties(final KylinConfig kylinConfig, StringBuilder hiveCmd) {
+ final Map<String, String> hiveConfOverride = kylinConfig.getHiveConfigOverride();
+ if (hiveConfOverride.isEmpty() == false) {
+ for (String key : hiveConfOverride.keySet()) {
+ hiveCmd.append("SET ").append(key).append("=").append(hiveConfOverride.get(key)).append(";\n");
+ }
+ }
+ }
+
+ private static void appendHiveOverrideProperties2(final KylinConfig kylinConfig, HiveCmdBuilder hiveCmdBuilder) {
+ final Map<String, String> hiveConfOverride = kylinConfig.getHiveConfigOverride();
+ if (hiveConfOverride.isEmpty() == false) {
+ for (String key : hiveConfOverride.keySet()) {
+ hiveCmdBuilder.addStatement("SET " + key + "=" + hiveConfOverride.get(key) + ";\n");
+ }
+ }
+ }
}
[04/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming
code
Posted by li...@apache.org.
KYLIN-2072 Cleanup old streaming code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5aee0226
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5aee0226
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5aee0226
Branch: refs/heads/master-cdh5.7
Commit: 5aee022612c6fa40c41e8c00063714b79b6d5237
Parents: cb2b12b
Author: shaofengshi <sh...@apache.org>
Authored: Sun Oct 9 13:10:50 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 10 13:32:44 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 4 -
.../kylin/job/streaming/KafkaDataLoader.java | 79 ----
build/bin/cleanup_streaming_files.sh | 42 --
build/bin/kylin.sh | 61 ---
build/bin/streaming_build.sh | 33 --
build/bin/streaming_check.sh | 29 --
build/bin/streaming_fillgap.sh | 26 --
build/bin/streaming_rolllog.sh | 29 --
.../metadata/streaming/StreamingConfig.java | 85 ++++
.../metadata/streaming/StreamingManager.java | 248 ++++++++++++
.../.settings/org.eclipse.core.resources.prefs | 6 -
.../.settings/org.eclipse.jdt.core.prefs | 386 -------------------
.../.settings/org.eclipse.jdt.ui.prefs | 7 -
engine-streaming/pom.xml | 121 ------
.../kylin/engine/streaming/BootstrapConfig.java | 71 ----
.../kylin/engine/streaming/IStreamingInput.java | 30 --
.../engine/streaming/IStreamingOutput.java | 34 --
.../streaming/OneOffStreamingBuilder.java | 71 ----
.../engine/streaming/StreamingBatchBuilder.java | 43 ---
.../kylin/engine/streaming/StreamingConfig.java | 85 ----
.../engine/streaming/StreamingManager.java | 248 ------------
.../kylin/engine/streaming/cli/MonitorCLI.java | 88 -----
.../engine/streaming/cli/StreamingCLI.java | 114 ------
.../streaming/cube/StreamingCubeBuilder.java | 168 --------
.../diagnose/StreamingLogAnalyzer.java | 96 -----
.../streaming/monitor/StreamingMonitor.java | 172 ---------
.../engine/streaming/util/StreamingUtils.java | 51 ---
.../kylin/provision/BuildCubeWithStream.java | 4 +-
pom.xml | 6 -
.../rest/controller/StreamingController.java | 2 +-
.../kylin/rest/controller/TableController.java | 2 +-
.../apache/kylin/rest/service/BasicService.java | 2 +-
.../kylin/rest/service/StreamingService.java | 2 +-
source-kafka/pom.xml | 6 -
.../kafka/ByteBufferBackedInputStream.java | 52 ---
.../apache/kylin/source/kafka/KafkaSource.java | 2 +-
.../kylin/source/kafka/KafkaStreamingInput.java | 227 -----------
.../source/kafka/TimedJsonStreamParser.java | 1 +
.../kafka/diagnose/KafkaInputAnalyzer.java | 312 ---------------
.../source/kafka/diagnose/KafkaVerify.java | 101 -----
.../source/kafka/diagnose/TimeHistogram.java | 85 ----
.../kafka/util/ByteBufferBackedInputStream.java | 52 +++
.../kylin/source/kafka/util/KafkaRequester.java | 191 ---------
.../kylin/source/kafka/util/KafkaUtils.java | 173 ---------
.../test/java/TimedJsonStreamParserTest.java | 4 +-
storage-hbase/pom.xml | 4 -
.../hbase/steps/HBaseStreamingOutput.java | 98 -----
.../apache/kylin/tool/CubeMetaExtractor.java | 4 +-
48 files changed, 397 insertions(+), 3360 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0c80afc..e6f83a8 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -47,10 +47,6 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-mr</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-streaming</artifactId>
- </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java b/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
deleted file mode 100644
index 454f6cf..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.streaming;
-
-import java.util.List;
-import java.util.Properties;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.source.kafka.config.BrokerConfig;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
-
-/**
- * Load prepared data into kafka(for test use)
- */
-public class KafkaDataLoader extends StreamDataLoader {
- List<KafkaClusterConfig> kafkaClusterConfigs;
-
- public KafkaDataLoader(KafkaConfig kafkaConfig) {
- super(kafkaConfig);
- this.kafkaClusterConfigs = kafkaConfig.getKafkaClusterConfigs();
- }
-
- public void loadIntoKafka(List<String> messages) {
-
- KafkaClusterConfig clusterConfig = kafkaClusterConfigs.get(0);
- String brokerList = StringUtils.join(Collections2.transform(clusterConfig.getBrokerConfigs(), new Function<BrokerConfig, String>() {
- @Nullable
- @Override
- public String apply(BrokerConfig brokerConfig) {
- return brokerConfig.getHost() + ":" + brokerConfig.getPort();
- }
- }), ",");
- Properties props = new Properties();
- props.put("metadata.broker.list", brokerList);
- props.put("serializer.class", "kafka.serializer.StringEncoder");
- props.put("request.required.acks", "1");
- props.put("retry.backoff.ms", "1000");
-
- ProducerConfig config = new ProducerConfig(props);
-
- Producer<String, String> producer = new Producer<String, String>(config);
-
- List<KeyedMessage<String, String>> keyedMessages = Lists.newArrayList();
- for (int i = 0; i < messages.size(); ++i) {
- KeyedMessage<String, String> keyedMessage = new KeyedMessage<>(clusterConfig.getTopic(), String.valueOf(i), messages.get(i));
- keyedMessages.add(keyedMessage);
- }
- producer.send(keyedMessages);
- producer.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/build/bin/cleanup_streaming_files.sh
----------------------------------------------------------------------
diff --git a/build/bin/cleanup_streaming_files.sh b/build/bin/cleanup_streaming_files.sh
deleted file mode 100644
index 9b31a4f..0000000
--- a/build/bin/cleanup_streaming_files.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [ $# != 1 ]
-then
- echo 'invalid input'
- exit -1
-fi
-
-cd $KYLIN_HOME/logs
-
-for pidfile in `find -L . -name "$1_1*"`
-do
- pidfile=`echo "$pidfile" | cut -c 3-`
- echo "pidfile:$pidfile"
- pid=`cat $pidfile`
- if [ `ps -ef | awk '{print $2}' | grep -w $pid | wc -l` = 1 ]
- then
- echo "pid:$pid still running"
- else
- echo "pid:$pid not running, try to delete files"
- echo $pidfile | xargs rm
- echo "streaming_$pidfile.log" | xargs rm
- fi
-done
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index e767492..039be9f 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -139,67 +139,6 @@ then
exit 1
fi
-# streaming command
-elif [ "$1" == "streaming" ]
-then
- if [ $# -lt 4 ]
- then
- echo "invalid input args $@"
- exit -1
- fi
- if [ "$2" == "start" ]
- then
- retrieveDependency
- source ${dir}/find-kafka-dependency.sh
-
- # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
- hbase ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=kylin-log4j.properties\
- -Dkylin.hive.dependency=${hive_dependency} \
- -Dkylin.kafka.dependency=${kafka_dependency} \
- -Dkylin.hbase.dependency=${hbase_dependency} \
- org.apache.kylin.engine.streaming.cli.StreamingCLI $@ > ${KYLIN_HOME}/logs/streaming_$3_$4.log 2>&1 & echo $! > ${KYLIN_HOME}/logs/$3_$4 &
- echo "streaming started name: $3 id: $4"
- exit 0
- elif [ "$2" == "stop" ]
- then
- if [ ! -f "${KYLIN_HOME}/$3_$4" ]
- then
- echo "streaming is not running, please check"
- exit 1
- fi
- pid=`cat ${KYLIN_HOME}/$3_$4`
- if [ "$pid" = "" ]
- then
- echo "streaming is not running, please check"
- exit 1
- else
- echo "stopping streaming:$pid"
- kill $pid
- fi
- rm ${KYLIN_HOME}/$3_$4
- exit 0
- else
- echo
- fi
-
-# monitor command
-elif [ "$1" == "monitor" ]
-then
- echo "monitor job"
-
- retrieveDependency
- source ${dir}/find-kafka-dependency.sh
-
- # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
- hbase ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=kylin-log4j.properties\
- -Dkylin.hive.dependency=${hive_dependency} \
- -Dkylin.kafka.dependency=${kafka_dependency} \
- -Dkylin.hbase.dependency=${hbase_dependency} \
- org.apache.kylin.engine.streaming.cli.MonitorCLI $@ > ${KYLIN_HOME}/logs/monitor.log 2>&1
- exit 0
-
elif [ "$1" = "version" ]
then
exec hbase -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.common.KylinVersion
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/build/bin/streaming_build.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_build.sh b/build/bin/streaming_build.sh
deleted file mode 100644
index ed19036..0000000
--- a/build/bin/streaming_build.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source /etc/profile
-source ~/.bash_profile
-
-CUBE=$1
-INTERVAL=$2
-DELAY=$3
-CURRENT_TIME_IN_SECOND=`date +%s`
-CURRENT_TIME=$((CURRENT_TIME_IN_SECOND * 1000))
-START=$(($CURRENT_TIME - CURRENT_TIME%INTERVAL - DELAY))
-END=$(($CURRENT_TIME - CURRENT_TIME%INTERVAL - DELAY + INTERVAL))
-
-ID="$START"_"$END"
-echo "building for ${ID}" >> ${KYLIN_HOME}/logs/build_trace.log
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${CUBE} ${ID} -start ${START} -end ${END} -cube ${CUBE}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/build/bin/streaming_check.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_check.sh b/build/bin/streaming_check.sh
deleted file mode 100644
index fef0139..0000000
--- a/build/bin/streaming_check.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source /etc/profile
-source ~/.bash_profile
-
-receivers=$1
-host=$2
-tablename=$3
-authorization=$4
-projectname=$5
-cubename=$6
-sh ${KYLIN_HOME}/bin/kylin.sh monitor -receivers ${receivers} -host ${host} -tableName ${tablename} -authorization ${authorization} -cubeName ${cubename} -projectName ${projectname}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/build/bin/streaming_fillgap.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_fillgap.sh b/build/bin/streaming_fillgap.sh
deleted file mode 100644
index c67809a..0000000
--- a/build/bin/streaming_fillgap.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source /etc/profile
-source ~/.bash_profile
-
-cube=$1
-
-cd ${KYLIN_HOME}
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${cube} fillgap -cube ${cube} -fillGap true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/build/bin/streaming_rolllog.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_rolllog.sh b/build/bin/streaming_rolllog.sh
deleted file mode 100644
index 8018eb8..0000000
--- a/build/bin/streaming_rolllog.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source /etc/profile
-source ~/.bash_profile
-
-KYLIN_LOG_HOME=${KYLIN_HOME}/logs
-cd ${KYLIN_LOG_HOME}
-timestamp=`date +%Y_%m_%d_%H_%M_%S`
-tarfile=logs_archived_at_${timestamp}.tar
-files=`find -L . ! -name '*.tar' -type f -mtime +1` # keep two days' log
-echo ${files} | xargs tar -cvf ${tarfile}
-echo ${files} | xargs rm
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingConfig.java b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingConfig.java
new file mode 100644
index 0000000..9fd6ede
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingConfig.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.streaming;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.persistence.Serializer;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class StreamingConfig extends RootPersistentEntity {
+
+ public static Serializer<StreamingConfig> SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class);
+
+ public static final String STREAMING_TYPE_KAFKA = "kafka";
+
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("type")
+ private String type = STREAMING_TYPE_KAFKA;
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getResourcePath() {
+ return concatResourcePath(name);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public static String concatResourcePath(String name) {
+ return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
+ }
+
+ @Override
+ public StreamingConfig clone() {
+ try {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ SERIALIZER.serialize(this, new DataOutputStream(baos));
+ return SERIALIZER.deserialize(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
+ } catch (IOException e) {
+ throw new RuntimeException(e);//in mem, should not happen
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
new file mode 100644
index 0000000..8cfe87d
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.streaming;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
+import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class StreamingManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(StreamingManager.class);
+
+ // static cached instances
+ private static final ConcurrentHashMap<KylinConfig, StreamingManager> CACHE = new ConcurrentHashMap<KylinConfig, StreamingManager>();
+
+ public static final Serializer<StreamingConfig> STREAMING_SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class);
+
+ private KylinConfig config;
+
+ // name ==> StreamingConfig
+ private CaseInsensitiveStringCache<StreamingConfig> streamingMap;
+
+ public static void clearCache() {
+ CACHE.clear();
+ }
+
+ private StreamingManager(KylinConfig config) throws IOException {
+ this.config = config;
+ this.streamingMap = new CaseInsensitiveStringCache<StreamingConfig>(config, "streaming");
+
+ // touch lower level metadata before registering my listener
+ reloadAllStreaming();
+ Broadcaster.getInstance(config).registerListener(new StreamingSyncListener(), "streaming");
+ }
+
+ private class StreamingSyncListener extends Broadcaster.Listener {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ clearCache();
+ }
+
+ @Override
+ public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException {
+ if (event == Event.DROP)
+ removeStreamingLocal(cacheKey);
+ else
+ reloadStreamingConfigLocal(cacheKey);
+ }
+ }
+
+ private ResourceStore getStore() {
+ return ResourceStore.getStore(this.config);
+ }
+
+ public static StreamingManager getInstance(KylinConfig config) {
+ StreamingManager r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+
+ synchronized (StreamingManager.class) {
+ r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+ try {
+ r = new StreamingManager(config);
+ CACHE.put(config, r);
+ if (CACHE.size() > 1) {
+ logger.warn("More than one singleton exist");
+ }
+ return r;
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to init StreamingManager from " + config, e);
+ }
+ }
+ }
+
+ private static String formatStreamingConfigPath(String name) {
+ return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
+ }
+
+ private static String formatStreamingOutputPath(String streaming, int partition) {
+ return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + partition + ".json";
+ }
+
+ private static String formatStreamingOutputPath(String streaming, List<Integer> partitions) {
+ return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + StringUtils.join(partitions, "_") + ".json";
+ }
+
+ public StreamingConfig getStreamingConfig(String name) {
+ return streamingMap.get(name);
+ }
+
+ public List<StreamingConfig> listAllStreaming() {
+ return new ArrayList<>(streamingMap.values());
+ }
+
+ /**
+ * Reload StreamingConfig from resource store It will be triggered by an desc
+ * update event.
+ *
+ * @param name
+ * @throws IOException
+ */
+ public StreamingConfig reloadStreamingConfigLocal(String name) throws IOException {
+
+ // Save Source
+ String path = StreamingConfig.concatResourcePath(name);
+
+ // Reload the StreamingConfig
+ StreamingConfig ndesc = loadStreamingConfigAt(path);
+
+ // Here replace the old one
+ streamingMap.putLocal(ndesc.getName(), ndesc);
+ return ndesc;
+ }
+
+ // remove streamingConfig
+ public void removeStreamingConfig(StreamingConfig streamingConfig) throws IOException {
+ String path = streamingConfig.getResourcePath();
+ getStore().deleteResource(path);
+ streamingMap.remove(streamingConfig.getName());
+ }
+
+ public StreamingConfig getConfig(String name) {
+ name = name.toUpperCase();
+ return streamingMap.get(name);
+ }
+
+ public void removeStreamingLocal(String streamingName) {
+ streamingMap.removeLocal(streamingName);
+ }
+
+ /**
+ * Update CubeDesc with the input. Broadcast the event into cluster
+ *
+ * @param desc
+ * @return
+ * @throws IOException
+ */
+ public StreamingConfig updateStreamingConfig(StreamingConfig desc) throws IOException {
+ // Validate CubeDesc
+ if (desc.getUuid() == null || desc.getName() == null) {
+ throw new IllegalArgumentException("SteamingConfig Illegal.");
+ }
+ String name = desc.getName();
+ if (!streamingMap.containsKey(name)) {
+ throw new IllegalArgumentException("StreamingConfig '" + name + "' does not exist.");
+ }
+
+ // Save Source
+ String path = desc.getResourcePath();
+ getStore().putResource(path, desc, STREAMING_SERIALIZER);
+
+ // Reload the StreamingConfig
+ StreamingConfig ndesc = loadStreamingConfigAt(path);
+ // Here replace the old one
+ streamingMap.put(ndesc.getName(), desc);
+
+ return ndesc;
+ }
+
+ public StreamingConfig saveStreamingConfig(StreamingConfig streamingConfig) throws IOException {
+ if (streamingConfig == null || StringUtils.isEmpty(streamingConfig.getName())) {
+ throw new IllegalArgumentException();
+ }
+
+ if (streamingMap.containsKey(streamingConfig.getName()))
+ throw new IllegalArgumentException("StreamingConfig '" + streamingConfig.getName() + "' already exists");
+
+ String path = StreamingConfig.concatResourcePath(streamingConfig.getName());
+ getStore().putResource(path, streamingConfig, StreamingConfig.SERIALIZER);
+ streamingMap.put(streamingConfig.getName(), streamingConfig);
+ return streamingConfig;
+ }
+
+ private StreamingConfig loadStreamingConfigAt(String path) throws IOException {
+ ResourceStore store = getStore();
+ StreamingConfig streamingDesc = store.getResource(path, StreamingConfig.class, STREAMING_SERIALIZER);
+
+ if (StringUtils.isBlank(streamingDesc.getName())) {
+ throw new IllegalStateException("StreamingConfig name must not be blank");
+ }
+ return streamingDesc;
+ }
+
+ private void reloadAllStreaming() throws IOException {
+ ResourceStore store = getStore();
+ logger.info("Reloading Streaming Metadata from folder " + store.getReadableResourcePath(ResourceStore.STREAMING_RESOURCE_ROOT));
+
+ streamingMap.clear();
+
+ List<String> paths = store.collectResourceRecursively(ResourceStore.STREAMING_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
+ for (String path : paths) {
+ StreamingConfig streamingConfig;
+ try {
+ streamingConfig = loadStreamingConfigAt(path);
+ } catch (Exception e) {
+ logger.error("Error loading streaming desc " + path, e);
+ continue;
+ }
+ if (path.equals(streamingConfig.getResourcePath()) == false) {
+ logger.error("Skip suspicious desc at " + path + ", " + streamingConfig + " should be at " + streamingConfig.getResourcePath());
+ continue;
+ }
+ if (streamingMap.containsKey(streamingConfig.getName())) {
+ logger.error("Dup StreamingConfig name '" + streamingConfig.getName() + "' on path " + path);
+ continue;
+ }
+
+ streamingMap.putLocal(streamingConfig.getName(), streamingConfig);
+ }
+
+ logger.debug("Loaded " + streamingMap.size() + " StreamingConfig(s)");
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/engine-streaming/.settings/org.eclipse.core.resources.prefs b/engine-streaming/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 29abf99..0000000
--- a/engine-streaming/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/engine-streaming/.settings/org.eclipse.jdt.core.prefs b/engine-streaming/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 5aaaf1e..0000000
--- a/engine-streaming/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,386 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=ignore
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=ignore
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/engine-streaming/.settings/org.eclipse.jdt.ui.prefs b/engine-streaming/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index d521bab..0000000
--- a/engine-streaming/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=99
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
deleted file mode 100644
index 876279d..0000000
--- a/engine-streaming/pom.xml
+++ /dev/null
@@ -1,121 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kylin-engine-streaming</artifactId>
- <packaging>jar</packaging>
- <name>Apache Kylin - Streaming Engine</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.6.0-SNAPSHOT</version>
-
- </parent>
-
- <properties>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-cube</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-storage</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-job</artifactId>
- </dependency>
-
- <!-- Env & Test -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- <!-- protobuf version conflict with hbase -->
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-app</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <classifier>hadoop2</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
deleted file mode 100644
index 35bdfa8..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.streaming;
-
-/**
- */
-public class BootstrapConfig {
-
- private String cubeName;
- private long start = 0L;
- private long end = 0L;
-
- private boolean fillGap;
- private long maxFillGapRange = 4 * 3600 * 1000L;
-
- public long getStart() {
- return start;
- }
-
- public void setStart(long start) {
- this.start = start;
- }
-
- public long getEnd() {
- return end;
- }
-
- public void setEnd(long end) {
- this.end = end;
- }
-
- public String getCubeName() {
- return cubeName;
- }
-
- public void setCubeName(String cubeName) {
- this.cubeName = cubeName;
- }
-
- public boolean isFillGap() {
- return fillGap;
- }
-
- public void setFillGap(boolean fillGap) {
- this.fillGap = fillGap;
- }
-
- public long getMaxFillGapRange() {
- return maxFillGapRange;
- }
-
- public void setMaxFillGapRange(long maxFillGapRange) {
- this.maxFillGapRange = maxFillGapRange;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
deleted file mode 100644
index c583283..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.metadata.realization.RealizationType;
-
-/**
- */
-public interface IStreamingInput {
-
- StreamingBatch getBatchWithTimeWindow(RealizationType realizationType, String realizationName, int id, long startTime, long endTime);
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingOutput.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingOutput.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingOutput.java
deleted file mode 100644
index cb15e2b..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingOutput.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.util.Map;
-
-import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.IBuildable;
-
-/**
- */
-public interface IStreamingOutput {
-
- ICuboidWriter getCuboidWriter(IBuildable buildable);
-
- void output(IBuildable buildable, Map<Long, HyperLogLogPlusCounter> samplingResult);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
deleted file mode 100644
index c9da46e..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.util.Map;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.engine.streaming.util.StreamingUtils;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.IBuildable;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationType;
-
-import com.google.common.base.Preconditions;
-
-/**
- */
-public class OneOffStreamingBuilder {
-
- private final IStreamingInput streamingInput;
- private final IStreamingOutput streamingOutput;
- private final StreamingBatchBuilder streamingBatchBuilder;
- private final long startTime;
- private final long endTime;
- private final RealizationType realizationType;
- private final String realizationName;
-
- public OneOffStreamingBuilder(RealizationType realizationType, String realizationName, long startTime, long endTime) {
- Preconditions.checkArgument(startTime < endTime);
- this.startTime = startTime;
- this.endTime = endTime;
- this.realizationType = Preconditions.checkNotNull(realizationType);
- this.realizationName = Preconditions.checkNotNull(realizationName);
- this.streamingInput = Preconditions.checkNotNull(StreamingUtils.getStreamingInput());
- this.streamingOutput = Preconditions.checkNotNull(StreamingUtils.getStreamingOutput());
- this.streamingBatchBuilder = Preconditions.checkNotNull(StreamingUtils.getMicroBatchBuilder(realizationType, realizationName));
- }
-
- public Runnable build() {
- return new Runnable() {
- @Override
- public void run() {
- StreamingBatch streamingBatch = streamingInput.getBatchWithTimeWindow(realizationType, realizationName, -1, startTime, endTime);
- final IBuildable buildable = streamingBatchBuilder.createBuildable(streamingBatch);
- final Map<Long, HyperLogLogPlusCounter> samplingResult = streamingBatchBuilder.sampling(streamingBatch);
- final Map<TblColRef, Dictionary<String>> dictionaryMap = streamingBatchBuilder.buildDictionary(streamingBatch, buildable);
- streamingBatchBuilder.build(streamingBatch, dictionaryMap, streamingOutput.getCuboidWriter(buildable));
- streamingOutput.output(buildable, samplingResult);
- streamingBatchBuilder.commit(buildable);
- }
- };
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
deleted file mode 100644
index 8b0b8e6..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.util.Map;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.IBuildable;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- */
-public interface StreamingBatchBuilder {
-
- IBuildable createBuildable(StreamingBatch streamingBatch);
-
- Map<Long, HyperLogLogPlusCounter> sampling(StreamingBatch streamingBatch);
-
- Map<TblColRef, Dictionary<String>> buildDictionary(StreamingBatch streamingBatch, IBuildable buildable);
-
- void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter);
-
- void commit(IBuildable buildable);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
deleted file mode 100644
index 9d1a0b1..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.persistence.Serializer;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class StreamingConfig extends RootPersistentEntity {
-
- public static Serializer<StreamingConfig> SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class);
-
- public static final String STREAMING_TYPE_KAFKA = "kafka";
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("type")
- private String type = STREAMING_TYPE_KAFKA;
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public static String concatResourcePath(String name) {
- return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
- }
-
- @Override
- public StreamingConfig clone() {
- try {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- SERIALIZER.serialize(this, new DataOutputStream(baos));
- return SERIALIZER.deserialize(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
- } catch (IOException e) {
- throw new RuntimeException(e);//in mem, should not happen
- }
- }
-
-}
[32/50] [abbrv] kylin git commit: minor,
mv method from query controller to service
Posted by li...@apache.org.
minor, mv method from query controller to service
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44cf9fba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44cf9fba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44cf9fba
Branch: refs/heads/master-cdh5.7
Commit: 44cf9fba5900252fb3b4364d4bd129d83212d51c
Parents: ef44b7f
Author: Roger Shi <ro...@kyligence.io>
Authored: Mon Oct 17 13:21:37 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Oct 17 13:33:08 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/QueryController.java | 136 +------------------
.../apache/kylin/rest/service/CacheService.java | 5 +-
.../apache/kylin/rest/service/QueryService.java | 132 +++++++++++++++++-
.../rest/controller/QueryControllerTest.java | 2 +-
4 files changed, 134 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/44cf9fba/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 9471937..c5f896d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -23,15 +23,10 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
-import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.model.SelectedColumnMeta;
import org.apache.kylin.rest.model.TableMeta;
@@ -41,12 +36,9 @@ import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryUtil;
-import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
@@ -58,12 +50,6 @@ import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
-import com.google.common.base.Preconditions;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
/**
* Handle query requests.
*
@@ -74,31 +60,20 @@ public class QueryController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(QueryController.class);
- public static final String SUCCESS_QUERY_CACHE = "StorageCache";
- public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
-
@Autowired
private QueryService queryService;
- @Autowired
- private CacheManager cacheManager;
-
- @PostConstruct
- public void init() throws IOException {
- Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
- }
-
@RequestMapping(value = "/query", method = RequestMethod.POST)
@ResponseBody
public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
- return doQueryWithCache(sqlRequest);
+ return queryService.doQueryWithCache(sqlRequest);
}
// TODO should be just "prepare" a statement, get back expected ResultSetMetaData
@RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
@ResponseBody
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
- return doQueryWithCache(sqlRequest);
+ return queryService.doQueryWithCache(sqlRequest);
}
@RequestMapping(value = "/saved_queries", method = RequestMethod.POST)
@@ -127,7 +102,7 @@ public class QueryController extends BasicController {
@RequestMapping(value = "/query/format/{format}", method = RequestMethod.GET)
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest sqlRequest, HttpServletResponse response) {
- SQLResponse result = doQueryWithCache(sqlRequest);
+ SQLResponse result = queryService.doQueryWithCache(sqlRequest);
response.setContentType("text/" + format + ";charset=utf-8");
response.setHeader("Content-Disposition", "attachment; filename=\"result." + format + "\"");
ICsvListWriter csvWriter = null;
@@ -164,112 +139,7 @@ public class QueryController extends BasicController {
}
}
- private SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
- try {
- BackdoorToggles.setToggles(sqlRequest.getBackdoorToggles());
-
- String sql = sqlRequest.getSql();
- String project = sqlRequest.getProject();
- logger.info("Using project: " + project);
- logger.info("The original query: " + sql);
-
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- String serverMode = kylinConfig.getServerMode();
- if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
- throw new InternalErrorException("Query is not allowed in " + serverMode + " mode.");
- }
-
- if (!sql.toLowerCase().contains("select")) {
- logger.debug("Directly return exception as not supported");
- throw new InternalErrorException("Not Supported SQL.");
- }
-
- long startTime = System.currentTimeMillis();
-
- SQLResponse sqlResponse = null;
- boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && !BackdoorToggles.getDisableCache();
- if (queryCacheEnabled) {
- sqlResponse = searchQueryInCache(sqlRequest);
- }
-
- try {
- if (null == sqlResponse) {
- sqlResponse = queryService.query(sqlRequest);
-
- long durationThreshold = kylinConfig.getQueryDurationCacheThreshold();
- long scancountThreshold = kylinConfig.getQueryScanCountCacheThreshold();
- sqlResponse.setDuration(System.currentTimeMillis() - startTime);
- logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
- String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
- if (queryCacheEnabled && !sqlResponse.getIsException() //
- && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
- cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
- }
- } else {
- sqlResponse.setDuration(System.currentTimeMillis() - startTime);
- }
-
- checkQueryAuth(sqlResponse);
-
- } catch (Throwable e) { // calcite may throw AssertError
- logger.error("Exception when execute sql", e);
- String errMsg = QueryUtil.makeErrorMsgUserFriendly(e);
-
- sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
-
- // for exception queries, only cache ScanOutOfLimitException
- if (queryCacheEnabled && e instanceof ScanOutOfLimitException) {
- Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
- exceptionCache.put(new Element(sqlRequest, sqlResponse));
- }
- }
-
- queryService.logQuery(sqlRequest, sqlResponse);
-
- QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
-
- if (sqlResponse.getIsException())
- throw new InternalErrorException(sqlResponse.getExceptionMessage());
-
- return sqlResponse;
-
- } finally {
- BackdoorToggles.cleanToggles();
- }
- }
-
- private SQLResponse searchQueryInCache(SQLRequest sqlRequest) {
- SQLResponse response = null;
- Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
- Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE);
-
- if (exceptionCache.get(sqlRequest) != null) {
- logger.info("The sqlResponse is found in EXCEPTION_QUERY_CACHE");
- Element element = exceptionCache.get(sqlRequest);
- response = (SQLResponse) element.getObjectValue();
- response.setHitExceptionCache(true);
- } else if (successCache.get(sqlRequest) != null) {
- logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE");
- Element element = successCache.get(sqlRequest);
- response = (SQLResponse) element.getObjectValue();
- response.setStorageCacheUsed(true);
- }
-
- return response;
- }
-
- private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
- if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
- queryService.checkAuthorization(sqlResponse.getCube());
- }
- }
-
public void setQueryService(QueryService queryService) {
this.queryService = queryService;
}
-
- public void setCacheManager(CacheManager cacheManager) {
- this.cacheManager = cacheManager;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/44cf9fba/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index c9c2dd7..0938e95 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -36,7 +36,6 @@ import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.kylin.rest.controller.QueryController;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -119,8 +118,8 @@ public class CacheService extends BasicService {
protected void cleanDataCache(String project) {
if (cacheManager != null) {
logger.info("cleaning cache for project" + project + " (currently remove all entries)");
- cacheManager.getCache(QueryController.SUCCESS_QUERY_CACHE).removeAll();
- cacheManager.getCache(QueryController.EXCEPTION_QUERY_CACHE).removeAll();
+ cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll();
+ cacheManager.getCache(QueryService.EXCEPTION_QUERY_CACHE).removeAll();
} else {
logger.warn("skip cleaning cache for project " + project);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/44cf9fba/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index cda4a52..a7ac4b6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -40,6 +40,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
@@ -51,6 +52,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DBUtils;
import org.apache.kylin.cube.CubeInstance;
@@ -60,6 +62,8 @@ import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.apache.kylin.rest.model.ColumnMeta;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.model.SelectedColumnMeta;
@@ -69,6 +73,7 @@ import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
@@ -80,8 +85,13 @@ import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+
/**
* @author xduo
*/
@@ -90,14 +100,14 @@ public class QueryService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(QueryService.class);
- @Autowired
- private CacheService cacheService;
-
public static final String USER_QUERY_FAMILY = "q";
private static final String DEFAULT_TABLE_PREFIX = "kylin_metadata";
private static final String USER_TABLE_NAME = "_user";
private static final String USER_QUERY_COLUMN = "c";
+ public static final String SUCCESS_QUERY_CACHE = "StorageCache";
+ public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
+
private final Serializer<Query[]> querySerializer = new Serializer<Query[]>(Query[].class);
private final BadQueryDetector badQueryDetector = new BadQueryDetector();
@@ -105,6 +115,17 @@ public class QueryService extends BasicService {
private final String tableNameBase;
private final String userTableName;
+ @Autowired
+ private CacheManager cacheManager;
+
+ @Autowired
+ private CacheService cacheService;
+
+ @PostConstruct
+ public void init() throws IOException {
+ Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
+ }
+
public QueryService() {
String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
// split TABLE@HBASE_URL
@@ -292,6 +313,106 @@ public class QueryService extends BasicService {
}
}
+ public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
+ try {
+ BackdoorToggles.setToggles(sqlRequest.getBackdoorToggles());
+
+ String sql = sqlRequest.getSql();
+ String project = sqlRequest.getProject();
+ logger.info("Using project: " + project);
+ logger.info("The original query: " + sql);
+
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ String serverMode = kylinConfig.getServerMode();
+ if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
+ throw new InternalErrorException("Query is not allowed in " + serverMode + " mode.");
+ }
+
+ if (!sql.toLowerCase().contains("select")) {
+ logger.debug("Directly return exception as not supported");
+ throw new InternalErrorException("Not Supported SQL.");
+ }
+
+ long startTime = System.currentTimeMillis();
+
+ SQLResponse sqlResponse = null;
+ boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && !BackdoorToggles.getDisableCache();
+ if (queryCacheEnabled) {
+ sqlResponse = searchQueryInCache(sqlRequest);
+ }
+
+ try {
+ if (null == sqlResponse) {
+ sqlResponse = query(sqlRequest);
+
+ long durationThreshold = kylinConfig.getQueryDurationCacheThreshold();
+ long scancountThreshold = kylinConfig.getQueryScanCountCacheThreshold();
+ sqlResponse.setDuration(System.currentTimeMillis() - startTime);
+ logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
+ String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
+ if (queryCacheEnabled && !sqlResponse.getIsException() //
+ && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
+ cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
+ }
+ } else {
+ sqlResponse.setDuration(System.currentTimeMillis() - startTime);
+ }
+
+ checkQueryAuth(sqlResponse);
+
+ } catch (Throwable e) { // calcite may throw AssertError
+ logger.error("Exception when execute sql", e);
+ String errMsg = QueryUtil.makeErrorMsgUserFriendly(e);
+
+ sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
+
+ // for exception queries, only cache ScanOutOfLimitException
+ if (queryCacheEnabled && e instanceof ScanOutOfLimitException) {
+ Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
+ exceptionCache.put(new Element(sqlRequest, sqlResponse));
+ }
+ }
+
+ logQuery(sqlRequest, sqlResponse);
+
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
+
+ if (sqlResponse.getIsException())
+ throw new InternalErrorException(sqlResponse.getExceptionMessage());
+
+ return sqlResponse;
+
+ } finally {
+ BackdoorToggles.cleanToggles();
+ }
+ }
+
+ public SQLResponse searchQueryInCache(SQLRequest sqlRequest) {
+ SQLResponse response = null;
+ Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
+ Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE);
+
+ if (exceptionCache.get(sqlRequest) != null) {
+ logger.info("The sqlResponse is found in EXCEPTION_QUERY_CACHE");
+ Element element = exceptionCache.get(sqlRequest);
+ response = (SQLResponse) element.getObjectValue();
+ response.setHitExceptionCache(true);
+ } else if (successCache.get(sqlRequest) != null) {
+ logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE");
+ Element element = successCache.get(sqlRequest);
+ response = (SQLResponse) element.getObjectValue();
+ response.setStorageCacheUsed(true);
+ }
+
+ return response;
+ }
+
+ private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
+ if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
+ checkAuthorization(sqlResponse.getCube());
+ }
+ }
+
private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
String userInfo = SecurityContextHolder.getContext().getAuthentication().getName();
final Collection<? extends GrantedAuthority> grantedAuthorities = SecurityContextHolder.getContext().getAuthentication().getAuthorities();
@@ -368,7 +489,7 @@ public class QueryService extends BasicService {
tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta);
}
}
-
+
} finally {
close(columnMeta, null, conn);
if (JDBCTableMeta != null) {
@@ -541,4 +662,7 @@ public class QueryService extends BasicService {
DBUtils.closeQuietly(conn);
}
+ public void setCacheManager(CacheManager cacheManager) {
+ this.cacheManager = cacheManager;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/44cf9fba/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
index 3180075..e84235b 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
@@ -49,7 +49,7 @@ public class QueryControllerTest extends ServiceTestBase {
queryController = new QueryController();
queryController.setQueryService(queryService);
- queryController.setCacheManager(cacheManager);
+ queryService.setCacheManager(cacheManager);
}
@Test(expected = Exception.class)
[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
Posted by li...@apache.org.
KYLIN-1672 support kylin on cdh 5.7
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f8b5e81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f8b5e81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f8b5e81
Branch: refs/heads/master-cdh5.7
Commit: 5f8b5e81614896dcd99bbb4b5fa106e764e101b8
Parents: b839156
Author: Lynne Jiang <ly...@hotmail.com>
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Oct 19 11:02:46 2016 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
.../kylin/engine/mr/steps/MockupMapContext.java | 15 +-
examples/test_case_data/sandbox/core-site.xml | 146 +++---
examples/test_case_data/sandbox/hbase-site.xml | 162 ++----
examples/test_case_data/sandbox/hdfs-site.xml | 259 ++--------
examples/test_case_data/sandbox/mapred-site.xml | 398 ++++++---------
examples/test_case_data/sandbox/yarn-site.xml | 496 ++-----------------
pom.xml | 16 +-
server/pom.xml | 36 ++
.../storage/hbase/steps/MockupMapContext.java | 19 +-
tool/pom.xml | 12 +
11 files changed, 428 insertions(+), 1131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
outKV[0] = key;
outKV[1] = value;
}
+
}
@Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
throw new NotImplementedException();
}
+
@Override
public float getProgress() {
throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
}
@Override
- public RawComparator<?> getSortComparator() {
+ public boolean userClassesTakesPrecedence() {
throw new NotImplementedException();
}
@Override
- public String getJar() {
+ public RawComparator<?> getSortComparator() {
throw new NotImplementedException();
}
@Override
- public RawComparator<?> getGroupingComparator() {
+ public String getJar() {
throw new NotImplementedException();
}
@@ -221,7 +223,7 @@ public class MockupMapContext {
@Override
public boolean getProfileEnabled() {
- throw new NotImplementedException();
+ return false;
}
@Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
public RawComparator<?> getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ return null;
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+<!--Autogenerated by Cloudera Manager-->
<configuration>
-
<property>
<name>fs.defaultFS</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
+ <value>hdfs://quickstart.cloudera:8020</value>
</property>
-
<property>
<name>fs.trash.interval</name>
- <value>360</value>
+ <value>1</value>
</property>
-
<property>
- <name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
- <value>120</value>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
-
<property>
- <name>hadoop.http.authentication.simple.anonymous.allowed</name>
- <value>true</value>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>DEFAULT</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.hosts</name>
+ <name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.hosts</name>
+ <name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.groups</name>
+ <name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.flume.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.flume.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.groups</name>
+ <value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hue.groups</name>
+ <name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.groups</name>
+ <name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.httpfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.groups</name>
+ <name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.hosts</name>
+ <name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.security.auth_to_local</name>
- <value>DEFAULT</value>
+ <name>hadoop.proxyuser.hdfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authentication</name>
- <value>simple</value>
+ <name>hadoop.proxyuser.yarn.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
+ <name>hadoop.proxyuser.yarn.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
-
<property>
- <name>io.file.buffer.size</name>
- <value>131072</value>
+ <name>hadoop.security.instrumentation.requires.admin</name>
+ <value>false</value>
</property>
-
<property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ <name>net.topology.script.file.name</name>
+ <value>/etc/hadoop/conf.cloudera.yarn/topology.py</value>
</property>
-
<property>
- <name>ipc.client.connect.max.retries</name>
- <value>50</value>
+ <name>io.file.buffer.size</name>
+ <value>65536</value>
</property>
-
<property>
- <name>ipc.client.connection.maxidletime</name>
- <value>30000</value>
+ <name>hadoop.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.client.idlethreshold</name>
- <value>8000</value>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.server.tcpnodelay</name>
- <value>true</value>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <final>true</final>
</property>
-
<property>
- <name>mapreduce.jobtracker.webinterface.trusted</name>
- <value>false</value>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <final>true</final>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 734908e..58c6223 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -15,180 +15,104 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
<property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
- </property>
-
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
+ <name>hbase.rootdir</name>
+ <value>hdfs://quickstart.cloudera:8020/hbase</value>
</property>
-
<property>
- <name>hbase.cluster.distributed</name>
+ <name>hbase.replication</name>
<value>true</value>
</property>
-
- <property>
- <name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
- </property>
-
<property>
- <name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <name>hbase.client.write.buffer</name>
+ <value>2097152</value>
</property>
-
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- </property>
-
<property>
- <name>hbase.hregion.majorcompaction</name>
- <value>604800000</value>
+ <name>hbase.client.pause</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.majorcompaction.jitter</name>
- <value>0.50</value>
+ <name>hbase.client.retries.number</name>
+ <value>35</value>
</property>
-
<property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
+ <name>hbase.ipc.client.allowsInterrupt</name>
<value>true</value>
</property>
-
<property>
- <name>hbase.hstore.blockingStoreFiles</name>
+ <name>hbase.client.primaryCallTimeout.get</name>
<value>10</value>
</property>
-
- <property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
- </property>
-
<property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
+ <name>hbase.client.primaryCallTimeout.multiget</name>
+ <value>10</value>
</property>
-
<property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
+ <name>hbase.regionserver.thrift.http</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.info.port</name>
- <value>60010</value>
+ <name>hbase.thrift.support.proxyuser</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.port</name>
+ <name>hbase.rpc.timeout</name>
<value>60000</value>
</property>
-
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
- </property>
-
<property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
+ <name>hbase.snapshot.enabled</name>
+ <value>true</value>
</property>
-
<property>
- <name>hbase.regionserver.info.port</name>
- <value>60030</value>
+ <name>hbase.snapshot.master.timeoutMillis</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rootdir</name>
- <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
+ <name>hbase.snapshot.region.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rpc.protection</name>
- <value>PRIVACY</value>
+ <name>hbase.snapshot.master.timeout.millis</name>
+ <value>60000</value>
</property>
-
<property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
-
<property>
- <name>hbase.security.authorization</name>
- <value>true</value>
+ <name>hbase.rpc.protection</name>
+ <value>authentication</value>
</property>
-
<property>
- <name>hbase.superuser</name>
- <value>hbase</value>
+ <name>zookeeper.session.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase</value>
</property>
-
<property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
+ <name>zookeeper.znode.rootserver</name>
+ <value>root-region-server</value>
</property>
-
<property>
<name>hbase.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com</value>
+ <value>quickstart.cloudera</value>
</property>
-
<property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- </property>
-
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
</property>
-
<property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
+ <name>hbase.rest.ssl.enabled</name>
+ <value>false</value>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1175fff..05854bd 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -15,271 +15,68 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.block.access.token.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>dfs.block.size</name>
- <value>34217472</value>
- </property>
-
- <property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
- </property>
-
- <property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
- </property>
-
- <property>
- <name>dfs.cluster.administrators</name>
- <value>hdfs</value>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- </property>
-
- <property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
- </property>
-
<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- </property>
-
- <property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- </property>
-
- <property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
+ <name>dfs.namenode.name.dir</name>
+ <value>file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
</property>
-
<property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value>quickstart.cloudera:8022</value>
</property>
-
<property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
+ <name>dfs.https.address</name>
+ <value>quickstart.cloudera:50470</value>
</property>
-
<property>
<name>dfs.https.port</name>
<value>50470</value>
</property>
-
- <property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/hadoop/hdfs/journalnode</value>
- </property>
-
- <property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
- </property>
-
- <property>
- <name>dfs.journalnode.https-address</name>
- <value>0.0.0.0:8481</value>
- </property>
-
- <property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
- </property>
-
<property>
<name>dfs.namenode.http-address</name>
- <value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.https-address</name>
- <value>sandbox.hortonworks.com:50470</value>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir</name>
- <value>/hadoop/hdfs/namenode</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <value>quickstart.cloudera:50070</value>
</property>
-
<property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>sandbox.hortonworks.com:50090</value>
+ <name>dfs.replication</name>
+ <value>1</value>
</property>
-
<property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
</property>
-
<property>
- <name>dfs.namenode.startup.delay.block.deletion.sec</name>
- <value>3600</value>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
</property>
-
<property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
+ <name>dfs.namenode.acls.enabled</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
+ <name>dfs.client.use.legacy.blockreader</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.replication</name>
- <value>1</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/run/hdfs-sockets/dn</value>
</property>
-
<property>
- <name>dfs.replication.max</name>
- <value>50</value>
+ <name>dfs.client.read.shortcircuit.skip.checksum</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.support.append</name>
- <value>true</value>
- <final>true</final>
+ <name>dfs.client.domain.socket.data.traffic</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.webhdfs.enabled</name>
+ <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
- <final>true</final>
</property>
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index e90f594..c9b1ca4 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -15,241 +15,165 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>io.sort.mb</name>
- <value>128</value>
- </property>
-
- <property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
- </property>
-
- <property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
- </property>
-
- <property>
- <name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
- </property>
-
- <property>
- <name>mapreduce.cluster.administrators</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-
- <property>
- <name>mapreduce.job.emit-timeline-data</name>
- <value>false</value>
- </property>
-
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>sandbox.hortonworks.com:10020</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>sandbox.hortonworks.com:19888</value>
- </property>
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
- </property>
-
- <property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
- </property>
-
- <property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
-
+<property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>10000000</value>
+</property>
+<property>
+ <name>mapreduce.job.counters.max</name>
+ <value>120</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress</name>
+ <value>true</value>
+</property>
+<property>
+ <name>zlib.compress.level</name>
+ <value>DEFAULT_COMPRESSION</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>64</value>
+</property>
+<property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>10</value>
+</property>
+<property>
+ <name>mapreduce.task.timeout</name>
+ <value>600000</value>
+</property>
+<property>
+ <name>mapreduce.client.submit.file.replication</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.reduces</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>16</value>
+</property>
+<property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>quickstart.cloudera:10020</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>quickstart.cloudera:19888</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.https.address</name>
+ <value>quickstart.cloudera:19890</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.admin.address</name>
+ <value>quickstart.cloudera:10033</value>
+</property>
+<property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+</property>
+<property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.ubertask.enable</name>
+ <value>false</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.map.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.reduce.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.heap.memory-mb.ratio</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.application.classpath</name>
+ <value>/tmp/kylin/*,/usr/lib/hadoop-mapreduce/lib/*,/etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-mapreduce/.//*,/usr/lib/hbase/hbase-common.jar,/etc/hive/conf</value>
+</property>
+<property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.shuffle.max.connections</name>
+ <value>80</value>
+</property>
</configuration>
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index 8256158..8988d4a 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -15,520 +15,128 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hadoop.registry.zk.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
<property>
<name>yarn.acl.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.http.policy</name>
- <value>HTTP_ONLY</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.root-dir</name>
- <value>/system/yarn/node-labels</value>
- </property>
-
- <property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>90</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
- <value>hadoop-yarn</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
- <value>30</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.dir</name>
- <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.enabled</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>8</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>9216</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
<property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>10</value>
+ <name>yarn.admin.acl</name>
+ <value>*</value>
</property>
-
<property>
<name>yarn.resourcemanager.address</name>
- <value>sandbox.hortonworks.com:8050</value>
+ <value>quickstart.cloudera:8032</value>
</property>
-
<property>
<name>yarn.resourcemanager.admin.address</name>
- <value>sandbox.hortonworks.com:8141</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.max-wait.ms</name>
- <value>900000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.retry-interval.ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.uri</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.ha.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.recovery.enabled</name>
- <value>true</value>
+ <value>quickstart.cloudera:8033</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>sandbox.hortonworks.com:8025</value>
- </property>
-
<property>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>sandbox.hortonworks.com:8030</value>
+ <value>quickstart.cloudera:8030</value>
</property>
-
<property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.state-store.max-completed-applications</name>
- <value>${yarn.resourcemanager.max-completed-applications}</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.store.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
- <value>true</value>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>quickstart.cloudera:8031</value>
</property>
-
<property>
<name>yarn.resourcemanager.webapp.address</name>
- <value>sandbox.hortonworks.com:8088</value>
+ <value>quickstart.cloudera:8088</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
- <value>false</value>
- </property>
-
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>localhost:8090</value>
+ <value>quickstart.cloudera:8090</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
- <value>*</value>
+ <name>yarn.resourcemanager.scheduler.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.admin.client.thread-count</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
- <value>*</value>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>true</value>
+ <name>yarn.scheduler.increment-allocation-mb</name>
+ <value>512</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
- <value>10000</value>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2816</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-acl</name>
- <value>world:anyone:rwcda</value>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
+ <name>yarn.scheduler.increment-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-num-retries</name>
- <value>1000</value>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name>
<value>1000</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-state-store.parent-path</name>
- <value>/rmstore</value>
+ <name>yarn.am.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.zk-timeout-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>9216</value>
- </property>
-
<property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1536</value>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.timeline-service.address</name>
- <value>sandbox.hortonworks.com:10200</value>
+ <name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name>
+ <value>1000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.max-retries</name>
- <value>30</value>
+ <name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.retry-interval-ms</name>
- <value>1000</value>
+ <name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*</value>
</property>
-
<property>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
-
<property>
- <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <name>yarn.scheduler.fair.user-as-default-queue</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.timeline-service.http-authentication.type</name>
- <value>simple</value>
- </property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>/hadoop/yarn/timeline</value>
+ <name>yarn.scheduler.fair.preemption</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
- <value>104857600</value>
+ <name>yarn.scheduler.fair.sizebasedweight</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
- <value>10000</value>
+ <name>yarn.scheduler.fair.assignmultiple</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <name>yarn.resourcemanager.max-completed-applications</name>
<value>10000</value>
</property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-ms</name>
- <value>2678400000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.address</name>
- <value>sandbox.hortonworks.com:8188</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8190</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3b666dd..63d314c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,19 +46,19 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
+ <yarn.version>2.6.0-cdh5.7.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>1.1.0-cdh5.7.0</hive.version>
+ <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
<curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
@@ -814,6 +814,10 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+ </repository>
</repositories>
<build>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 43453a8..d5f7ced 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -112,6 +112,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@@ -131,6 +135,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -146,6 +154,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -161,6 +173,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -176,6 +192,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -199,6 +219,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -214,6 +238,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -229,6 +257,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -262,6 +294,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
index d5c3f60..5adf327 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
@@ -100,11 +100,6 @@ public class MockupMapContext {
}
@Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
public Counter getCounter(Enum<?> counterName) {
throw new NotImplementedException();
}
@@ -165,6 +160,11 @@ public class MockupMapContext {
}
@Override
+ public boolean userClassesTakesPrecedence() {
+ return false;
+ }
+
+ @Override
public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
throw new NotImplementedException();
}
@@ -214,10 +214,6 @@ public class MockupMapContext {
throw new NotImplementedException();
}
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
@Override
public boolean getProfileEnabled() {
@@ -230,11 +226,6 @@ public class MockupMapContext {
}
@Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
public String getUser() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index e530469..72166c3 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -45,6 +45,18 @@
<!--Env-->
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
[24/50] [abbrv] kylin git commit: minor, enhance KYLIN_HOME setup
Posted by li...@apache.org.
minor, enhance KYLIN_HOME setup
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/021ccfd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/021ccfd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/021ccfd0
Branch: refs/heads/master-cdh5.7
Commit: 021ccfd0984bd41e2f2286ebec83efb551899575
Parents: bebbc78
Author: Yang Li <li...@apache.org>
Authored: Sat Oct 15 15:58:48 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Oct 15 15:59:38 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 7 ++++---
build/bin/metastore.sh | 9 +++++----
2 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/021ccfd0/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index da53d3d..9286055e 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -17,9 +17,10 @@
# limitations under the License.
#
-# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-KYLIN_HOME=`dirname $0`/..
+dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+
+# set KYLIN_HOME with consideration for multiple instances that are on the same node
+KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
dir="$KYLIN_HOME/bin"
http://git-wip-us.apache.org/repos/asf/kylin/blob/021ccfd0/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 5f34bff..baf3d7a 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -24,11 +24,12 @@
# take a look at SandboxMetastoreCLI
-dir=$(dirname ${0})
+dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
-# We should set KYLIN_HOME here for multiple tomcat instsances that are on the same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-export KYLIN_HOME=${dir}/../
+# set KYLIN_HOME with consideration for multiple instances that are on the same node
+KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
+export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
+dir="$KYLIN_HOME/bin"
source ${dir}/check-env.sh
[30/50] [abbrv] kylin git commit: KYLIN-2097 Get 'Column does not
exist in row key desc" on cube has TopN measure
Posted by li...@apache.org.
KYLIN-2097 Get 'Column does not exist in row key desc" on cube has TopN measure
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3bfd770
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3bfd770
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3bfd770
Branch: refs/heads/master-cdh5.7
Commit: c3bfd770af93db7e7c295846898851c4bdc458c5
Parents: 70c85a7
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 17 12:22:45 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 17 12:23:11 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3bfd770/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 33ab314..b67c374 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -246,6 +246,11 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (digest.groupbyColumns.containsAll(literalCol) == false)
return null;
+ for (TblColRef colRef : literalCol) {
+ if (digest.filterColumns.contains(colRef) == true) {
+ return null;
+ }
+ }
unmatchedDimensions.removeAll(literalCol);
unmatchedAggregations.remove(onlyFunction);
return new CapabilityInfluence() {
[07/50] [abbrv] kylin git commit: KYLIN-2078 Can't see generated SQL
at Web UI
Posted by li...@apache.org.
KYLIN-2078 Can't see generated SQL at Web UI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94a9f381
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94a9f381
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94a9f381
Branch: refs/heads/master-cdh5.7
Commit: 94a9f381548ff6ecfb253e189a3319252f18d16c
Parents: e23f761
Author: gaodayue <ga...@meituan.com>
Authored: Mon Oct 10 16:27:01 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Mon Oct 10 16:27:01 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller/CubeController.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/94a9f381/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index be242c3..bb7dc6a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -40,7 +40,6 @@ import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -142,8 +141,7 @@ public class CubeController extends BasicController {
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY);
- IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
+ IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
GeneralResponse repsonse = new GeneralResponse();
[33/50] [abbrv] kylin git commit: minor,
better name for hive build step
Posted by li...@apache.org.
minor, better name for hive build step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef0c9e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef0c9e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef0c9e5c
Branch: refs/heads/master-cdh5.7
Commit: ef0c9e5c64d5fbb1aebc415755b8f2b9ff8d1a83
Parents: 44cf9fb
Author: Li Yang <li...@apache.org>
Authored: Mon Oct 17 13:38:42 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Oct 17 13:39:06 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/constant/ExecutableConstants.java | 1 +
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0c9e5c/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 893c034..ad0b1b1 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -51,6 +51,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid Statistics";
public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
+ public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection";
public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS";
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0c9e5c/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 09ac522..202e480 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -244,7 +244,7 @@ public class HiveMRInput implements IMRInput {
@Override
public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
GarbageCollectionStep step = new GarbageCollectionStep();
- step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+ step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
step.setIntermediateTableIdentity(getIntermediateTableIdentity());
step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId())));
step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables);
[28/50] [abbrv] kylin git commit: KYLIN-2088 fix CI
Posted by li...@apache.org.
KYLIN-2088 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81403c1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81403c1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81403c1b
Branch: refs/heads/master-cdh5.7
Commit: 81403c1b879f23760d91e2fc0864ece17da00d00
Parents: 0018a21
Author: Yang Li <li...@apache.org>
Authored: Mon Oct 17 05:54:45 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Oct 17 05:54:45 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/query/ITKylinQueryTest.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/81403c1b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index a0706ca..ffd85e3 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -248,6 +248,14 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
+ public void testIntersectCountQuery() throws Exception {
+ // cannot compare coz H2 does not support intersect count yet..
+ if ("left".equalsIgnoreCase(joinType)) {
+ this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_intersect_count");
+ }
+ }
+
+ @Test
public void testDimDistinctCountQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
}
@@ -355,10 +363,4 @@ public class ITKylinQueryTest extends KylinTestBase {
this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_window");
}
- @Test
- public void testIntersectCountQuery() throws Exception {
- // cannot compare coz H2 does not support intersect count yet..
- this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_intersect_count");
- }
-
}
[20/50] [abbrv] kylin git commit: KYLIN-2012 adjust rules for
LookupTable & remove dead methods
Posted by li...@apache.org.
KYLIN-2012 adjust rules for LookupTable & remove dead methods
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36cf99ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36cf99ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36cf99ef
Branch: refs/heads/master-cdh5.7
Commit: 36cf99ef77486c1361a31f3e1f748bb277eca217
Parents: 96935fe
Author: gaodayue <ga...@meituan.com>
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Thu Oct 13 16:56:35 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 30 --------------------
.../apache/kylin/source/hive/SchemaChecker.java | 27 ++++++++++++++----
2 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 4e9c1ed..92d41b9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
}
- public void reloadTableCache(String tableIdentity) throws IOException {
- reloadSourceTableExt(tableIdentity);
- reloadSourceTable(tableIdentity);
- }
-
public DataModelDesc getDataModelDesc(String name) {
return dataModelDescMap.get(name);
}
@@ -622,29 +617,4 @@ public class MetadataManager {
srcTableExdMap.remove(tableIdentity);
}
- public String appendDBName(String table) {
-
- if (table.indexOf(".") > 0)
- return table;
-
- Map<String, TableDesc> map = getAllTablesMap();
-
- int count = 0;
- String result = null;
- for (TableDesc t : map.values()) {
- if (t.getName().equalsIgnoreCase(table)) {
- result = t.getIdentity();
- count++;
- }
- }
-
- if (count == 1)
- return result;
-
- if (count > 1) {
- logger.warn("There are more than 1 table named with '" + table + "' in different database; The program couldn't determine, randomly pick '" + result + "'");
- }
- return result;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index e955410..319ebee 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -152,7 +152,14 @@ public class SchemaChecker {
}
}
- private List<String> checkAllUsedColumns(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) {
+ /**
+ * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`.
+ * @param cube cube to check, must use `table` in its model
+ * @param table kylin's table metadata
+ * @param fieldsMap current hive schema of `table`
+ * @return true if all columns used in `cube` has compatible schema with `fieldsMap`, false otherwise
+ */
+ private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) {
Set<ColumnDesc> usedColumns = Sets.newHashSet();
for (TblColRef col : cube.getAllColumns()) {
usedColumns.add(col.getColumnDesc());
@@ -170,8 +177,15 @@ public class SchemaChecker {
return violateColumns;
}
- private boolean checkAllColumns(TableDesc table, List<FieldSchema> fields) {
- if (table.getColumnCount() != fields.size()) {
+ /**
+ * check whether all columns in `table` are still in `fields` and have the same index as before.
+ *
+ * @param table kylin's table metadata
+ * @param fields current table metadata in hive
+ * @return true if only new columns are appended in hive, false otherwise
+ */
+ private boolean checkAllColumnsInTableDesc(TableDesc table, List<FieldSchema> fields) {
+ if (table.getColumnCount() > fields.size()) {
return false;
}
@@ -212,15 +226,16 @@ public class SchemaChecker {
// if user reloads a fact table used by cube, then all used columns
// must match current schema
if (factTable.getIdentity().equals(fullTableName)) {
- List<String> violateColumns = checkAllUsedColumns(cube, factTable, currentFieldsMap);
+ List<String> violateColumns = checkAllColumnsInCube(cube, factTable, currentFieldsMap);
if (!violateColumns.isEmpty()) {
issues.add(format("Column %s used in cube[%s] and model[%s], but changed in hive", violateColumns, cube.getName(), modelName));
}
}
- // if user reloads a lookup table used by cube, then nearly all changes in schema are disallowed)
+ // if user reloads a lookup table used by cube, only append column(s) are allowed, all existing columns
+ // must be the same (except compatible type changes)
for (TableDesc lookupTable : lookupTables) {
- if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumns(lookupTable, currentFields)) {
+ if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumnsInTableDesc(lookupTable, currentFields)) {
issues.add(format("Table '%s' is used as Lookup Table in cube[%s] and model[%s], but changed in hive", lookupTable.getIdentity(), cube.getName(), modelName));
}
}
[38/50] [abbrv] kylin git commit: delete the blank line code
Posted by li...@apache.org.
delete the blank line code
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc2b59fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc2b59fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc2b59fa
Branch: refs/heads/master-cdh5.7
Commit: cc2b59faf1a12e76e6af83ba3a19fbe27ba54cc3
Parents: ac356f0
Author: lijieliang <li...@cmss.chinamobile.com>
Authored: Fri Oct 14 14:28:01 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 17 18:25:26 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc2b59fa/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 2ec1fbb..4ec8d3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -130,7 +130,6 @@ public class HiveMRInput implements IMRInput {
@Override
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) {
final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
-
final KylinConfig kylinConfig = CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
String createFlatTableMethod = kylinConfig.getCreateFlatHiveTableMethod();
[46/50] [abbrv] kylin git commit: KYLIN-1839 minor refactor
Posted by li...@apache.org.
KYLIN-1839 minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6cafc422
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6cafc422
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6cafc422
Branch: refs/heads/master-cdh5.7
Commit: 6cafc422bc0ca2fa19a4516bcb4388719c99184c
Parents: aa1550c
Author: Yang Li <li...@apache.org>
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Oct 19 08:30:49 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/AbstractHadoopJob.java | 13 +++----------
1 file changed, 3 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6cafc422/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index f70e3bb..a5b2d2e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -318,7 +318,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, fs, p);
+ appendTmpDir(job, fs, p, jarList, fileList);
continue;
}
@@ -335,18 +335,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
- private void appendTmpDir(Job job, FileSystem fs, Path tmpDir) {
+ private void appendTmpDir(Job job, FileSystem fs, Path tmpDir, StringBuilder jarList, StringBuilder fileList) {
try {
- Configuration jobConf = job.getConfiguration();
FileStatus[] fList = fs.listStatus(tmpDir);
- StringBuilder jarList = new StringBuilder();
- StringBuilder fileList = new StringBuilder();
-
for (FileStatus file : fList) {
Path p = file.getPath();
if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, fs, p);
+ appendTmpDir(job, fs, p, jarList, fileList);
continue;
}
@@ -356,9 +352,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
list.append(fs.getFileStatus(p).getPath().toString());
}
- appendTmpFiles(fileList.toString(), jobConf);
- appendTmpJars(jarList.toString(), jobConf);
-
} catch (IOException e) {
throw new RuntimeException(e);
}
[37/50] [abbrv] kylin git commit: hive job use overrided MR job
configuration by cube properties
Posted by li...@apache.org.
hive job use overrided MR job configuration by cube properties
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac356f01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac356f01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac356f01
Branch: refs/heads/master-cdh5.7
Commit: ac356f014d52c4b13ad72e9d6a537e50e9ace5fb
Parents: c92f79a
Author: lijieliang <li...@cmss.chinamobile.com>
Authored: Fri Oct 14 13:01:32 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 17 18:25:26 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/source/hive/HiveMRInput.java | 24 +++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ac356f01/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 202e480..2ec1fbb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -60,6 +61,8 @@ import com.google.common.collect.Sets;
public class HiveMRInput implements IMRInput {
+ private static final String MR_OVERRIDE_JOB_QUEUENAME = "mapreduce.job.queuename";
+
@Override
public IMRBatchCubingInputSide getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
return new BatchCubingInputSide(flatDesc);
@@ -154,7 +157,10 @@ public class HiveMRInput implements IMRInput {
StringBuilder hiveInitBuf = new StringBuilder();
hiveInitBuf.append("USE ").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
-
+ final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
+ if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+ hiveInitBuf.append("SET mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
+ }
String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
RedistributeFlatHiveTableStep step = new RedistributeFlatHiveTableStep();
@@ -172,6 +178,11 @@ public class HiveMRInput implements IMRInput {
final ShellExecutable step = new ShellExecutable();
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
+ if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+ hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+ kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
+ }
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
hiveCmdBuilder.addStatement("set hive.exec.compress.output=false;\n");
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc, rowCountOutputDir));
@@ -187,7 +198,7 @@ public class HiveMRInput implements IMRInput {
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
MetadataManager metadataManager = MetadataManager.getInstance(kylinConfig);
final Set<TableDesc> lookupViewsTables = Sets.newHashSet();
@@ -201,6 +212,10 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
+ if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+ hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+ kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
+ }
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";";
hiveCmdBuilder.addStatement(useDatabaseHql);
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
@@ -225,7 +240,10 @@ public class HiveMRInput implements IMRInput {
public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, boolean redistribute, String rowCountOutputDir) {
StringBuilder hiveInitBuf = new StringBuilder();
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
-
+ final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
+ if (kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+ hiveInitBuf.append("SET mapreduce.job.queuename =").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
+ }
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";\n";
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId));
[02/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming
code
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
deleted file mode 100644
index 1c579c6..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.diagnose;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class TimeHistogram {
- private long[] bucketsBoundary;
- private AtomicLong[] counters;
- private String id;
-
- private static Object printLock = new Object();
-
- /**
- * example: [10,20] will generate three buckets: (-\u221e,10), [10,20),[20,+\u221e)
- * unit: second
- */
- public TimeHistogram(long[] bucketsBoundary, String id) {
- this.bucketsBoundary = bucketsBoundary;
- this.counters = new AtomicLong[this.bucketsBoundary.length + 1];
- for (int i = 0; i < counters.length; i++) {
- this.counters[i] = new AtomicLong();
- }
- this.id = id;
- }
-
- /**
- * @param second in seconds
- */
- public void process(long second) {
- for (int i = 0; i < bucketsBoundary.length; ++i) {
- if (second < bucketsBoundary[i]) {
- counters[i].incrementAndGet();
- return;
- }
- }
-
- counters[bucketsBoundary.length].incrementAndGet();
- }
-
- /**
- * @param millis in milli seconds
- */
- public void processMillis(long millis) {
- process(millis / 1000);
- }
-
- public void printStatus() {
- long[] countersSnapshot = new long[counters.length];
- for (int i = 0; i < countersSnapshot.length; i++) {
- countersSnapshot[i] = counters[i].get();
- }
-
- long sum = 0;
- for (long counter : countersSnapshot) {
- sum += counter;
- }
-
- synchronized (printLock) {
- System.out.println("============== status of TimeHistogram " + id + " =================");
-
- for (int i = 0; i < countersSnapshot.length; ++i) {
- System.out.println(String.format("bucket: %d , count: %d ,percentage: %.4f", i, countersSnapshot[i], 1.0 * countersSnapshot[i] / (sum == 0 ? 1 : sum)));
- }
-
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
new file mode 100644
index 0000000..7a42598
--- /dev/null
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.kafka.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ */
+public class ByteBufferBackedInputStream extends InputStream {
+
+ private ByteBuffer buf;
+
+ public ByteBufferBackedInputStream(ByteBuffer buf) {
+ this.buf = buf;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (!buf.hasRemaining()) {
+ return -1;
+ }
+ return buf.get() & 0xFF;
+ }
+
+ @Override
+ public int read(byte[] bytes, int off, int len) throws IOException {
+ if (!buf.hasRemaining()) {
+ return -1;
+ }
+
+ len = Math.min(len, buf.remaining());
+ buf.get(bytes, off, len);
+ return len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaRequester.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaRequester.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaRequester.java
deleted file mode 100644
index bce9bb9..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaRequester.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.kafka.util;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-
-import javax.annotation.Nullable;
-
-import kafka.cluster.BrokerEndPoint;
-import org.apache.kafka.common.protocol.SecurityProtocol;
-import org.apache.kylin.source.kafka.TopicMeta;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import kafka.api.FetchRequestBuilder;
-import kafka.api.PartitionOffsetRequestInfo;
-import kafka.cluster.Broker;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.OffsetRequest;
-import kafka.javaapi.OffsetResponse;
-import kafka.javaapi.PartitionMetadata;
-import kafka.javaapi.TopicMetadata;
-import kafka.javaapi.TopicMetadataRequest;
-import kafka.javaapi.TopicMetadataResponse;
-import kafka.javaapi.consumer.SimpleConsumer;
-
-/**
- */
-public final class KafkaRequester {
-
- private static final Logger logger = LoggerFactory.getLogger(KafkaRequester.class);
-
- private static ConcurrentMap<String, SimpleConsumer> consumerCache = Maps.newConcurrentMap();
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
- @Override
- public void run() {
- KafkaRequester.shutdown();
- }
- }));
- }
-
- private static SimpleConsumer getSimpleConsumer(Broker broker, int timeout, int bufferSize, String clientId) {
- String key = createKey(broker, timeout, bufferSize, clientId);
- if (consumerCache.containsKey(key)) {
- return consumerCache.get(key);
- } else {
- BrokerEndPoint brokerEndPoint = broker.getBrokerEndPoint(SecurityProtocol.PLAINTEXT);
- consumerCache.putIfAbsent(key, new SimpleConsumer(brokerEndPoint.host(), brokerEndPoint.port(), timeout, bufferSize, clientId));
- return consumerCache.get(key);
- }
- }
-
- private static String createKey(Broker broker, int timeout, int bufferSize, String clientId) {
- return broker.getBrokerEndPoint(SecurityProtocol.PLAINTEXT).connectionString() + "_" + timeout + "_" + bufferSize + "_" + clientId;
- }
-
- public static TopicMeta getKafkaTopicMeta(KafkaClusterConfig kafkaClusterConfig) {
- SimpleConsumer consumer;
- for (Broker broker : kafkaClusterConfig.getBrokers()) {
- consumer = getSimpleConsumer(broker, kafkaClusterConfig.getTimeout(), kafkaClusterConfig.getBufferSize(), "topic_meta_lookup");
- List<String> topics = Collections.singletonList(kafkaClusterConfig.getTopic());
- TopicMetadataRequest req = new TopicMetadataRequest(topics);
- TopicMetadataResponse resp;
- try {
- resp = consumer.send(req);
- } catch (Exception e) {
- logger.warn("cannot send TopicMetadataRequest successfully: " + e);
- continue;
- }
- final List<TopicMetadata> topicMetadatas = resp.topicsMetadata();
- if (topicMetadatas.size() != 1) {
- break;
- }
- final TopicMetadata topicMetadata = topicMetadatas.get(0);
- if (topicMetadata.errorCode() != 0) {
- break;
- }
- List<Integer> partitionIds = Lists.transform(topicMetadata.partitionsMetadata(), new Function<PartitionMetadata, Integer>() {
- @Nullable
- @Override
- public Integer apply(PartitionMetadata partitionMetadata) {
- return partitionMetadata.partitionId();
- }
- });
- return new TopicMeta(kafkaClusterConfig.getTopic(), partitionIds);
- }
- logger.debug("cannot find topic:" + kafkaClusterConfig.getTopic());
- return null;
- }
-
- public static PartitionMetadata getPartitionMetadata(String topic, int partitionId, List<Broker> brokers, KafkaClusterConfig kafkaClusterConfig) {
- logger.debug("Brokers: " + brokers.toString());
- SimpleConsumer consumer;
- for (Broker broker : brokers) {
- consumer = getSimpleConsumer(broker, kafkaClusterConfig.getTimeout(), kafkaClusterConfig.getBufferSize(), "topic_meta_lookup");
- List<String> topics = Collections.singletonList(topic);
- TopicMetadataRequest req = new TopicMetadataRequest(topics);
- TopicMetadataResponse resp;
- try {
- resp = consumer.send(req);
- } catch (Exception e) {
- logger.warn("cannot send TopicMetadataRequest successfully: " + e);
- continue;
- }
- final List<TopicMetadata> topicMetadatas = resp.topicsMetadata();
- if (topicMetadatas.size() != 1) {
- logger.warn("invalid topicMetadata size:" + topicMetadatas.size());
- break;
- }
- final TopicMetadata topicMetadata = topicMetadatas.get(0);
- if (topicMetadata.errorCode() != 0) {
- logger.warn("fetching topicMetadata with errorCode:" + topicMetadata.errorCode());
- break;
- }
- for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
- StringBuffer logText = new StringBuffer();
- logText.append("PartitionMetadata debug errorCode: " + partitionMetadata.errorCode());
- logText.append("PartitionMetadata debug partitionId: " + partitionMetadata.partitionId());
- logText.append("PartitionMetadata debug leader: " + partitionMetadata.leader());
- logText.append("PartitionMetadata debug ISR: " + partitionMetadata.isr());
- logText.append("PartitionMetadata debug replica: " + partitionMetadata.replicas());
- logger.info(logText.toString());
- if (partitionMetadata.partitionId() == partitionId) {
- return partitionMetadata;
- }
- }
- }
- logger.debug("cannot find PartitionMetadata, topic:" + topic + " partitionId:" + partitionId);
- return null;
- }
-
- public static FetchResponse fetchResponse(String topic, int partitionId, long offset, Broker broker, KafkaClusterConfig kafkaClusterConfig) {
- final String clientName = "client_" + topic + "_" + partitionId;
- SimpleConsumer consumer = getSimpleConsumer(broker, kafkaClusterConfig.getTimeout(), kafkaClusterConfig.getBufferSize(), clientName);
- kafka.api.FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(topic, partitionId, offset, 1048576) // Note: this fetchSize of 100000 might need to be increased if large batches are written to Kafka, 1048576 is the default value on shell
- .build();
- return consumer.fetch(req);
- }
-
- public static long getLastOffset(String topic, int partitionId, long whichTime, Broker broker, KafkaClusterConfig kafkaClusterConfig) {
- String clientName = "client_" + topic + "_" + partitionId;
- SimpleConsumer consumer = getSimpleConsumer(broker, kafkaClusterConfig.getTimeout(), kafkaClusterConfig.getBufferSize(), clientName);
- TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
- Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
- requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
- OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
- OffsetResponse response = consumer.getOffsetsBefore(request);
-
- if (response.hasError()) {
- logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partitionId));
- return 0;
- }
- long[] offsets = response.offsets(topic, partitionId);
- return offsets[0];
- }
-
- public static void shutdown() {
- for (SimpleConsumer simpleConsumer : consumerCache.values()) {
- simpleConsumer.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
deleted file mode 100644
index ee5bb20..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.util;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.kafka.common.protocol.SecurityProtocol;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.source.kafka.StreamingParser;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-import kafka.api.OffsetRequest;
-import kafka.cluster.Broker;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.PartitionMetadata;
-import kafka.message.MessageAndOffset;
-
-/**
- */
-public final class KafkaUtils {
-
- private static final Logger logger = LoggerFactory.getLogger(KafkaUtils.class);
-
- private static final int MAX_RETRY_TIMES = 6;
-
- private KafkaUtils() {
- }
-
- public static Broker getLeadBroker(KafkaClusterConfig kafkaClusterConfig, int partitionId) {
- final PartitionMetadata partitionMetadata = KafkaRequester.getPartitionMetadata(kafkaClusterConfig.getTopic(), partitionId, kafkaClusterConfig.getBrokers(), kafkaClusterConfig);
- if (partitionMetadata != null) {
- if (partitionMetadata.errorCode() != 0) {
- logger.warn("PartitionMetadata errorCode: " + partitionMetadata.errorCode());
- }
- return new Broker(partitionMetadata.leader(), SecurityProtocol.PLAINTEXT);
- } else {
- return null;
- }
- }
-
- private static void sleep(int retryTimes) {
- int seconds = (int) Math.pow(2, retryTimes);
- logger.info("retry times:" + retryTimes + " sleep:" + seconds + " seconds");
- try {
- Thread.sleep(seconds * 1000);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static MessageAndOffset getKafkaMessage(KafkaClusterConfig kafkaClusterConfig, int partitionId, long offset) {
- final String topic = kafkaClusterConfig.getTopic();
- int retry = 0;
- while (retry < MAX_RETRY_TIMES) {//max sleep time 63 seconds
- final Broker leadBroker = getLeadBroker(kafkaClusterConfig, partitionId);
- if (leadBroker == null) {
- logger.warn("unable to find leadBroker with config:" + kafkaClusterConfig + " partitionId:" + partitionId);
- sleep(retry++);
- continue;
- }
- final FetchResponse response = KafkaRequester.fetchResponse(topic, partitionId, offset, leadBroker, kafkaClusterConfig);
- if (response.errorCode(topic, partitionId) != 0) {
- logger.warn("errorCode of FetchResponse is:" + response.errorCode(topic, partitionId));
- sleep(retry++);
- continue;
- }
- final Iterator<MessageAndOffset> iterator = response.messageSet(topic, partitionId).iterator();
- if (!iterator.hasNext()) {
- logger.warn("messageSet is empty");
- sleep(retry++);
- continue;
- }
- return iterator.next();
- }
- throw new IllegalStateException(String.format("try to get timestamp of topic: %s, partitionId: %d, offset: %d, failed to get StreamMessage from kafka", topic, partitionId, offset));
- }
-
- public static long findClosestOffsetWithDataTimestamp(KafkaClusterConfig kafkaClusterConfig, int partitionId, long timestamp, StreamingParser streamingParser) {
- Pair<Long, Long> firstAndLast = getFirstAndLastOffset(kafkaClusterConfig, partitionId);
- final String topic = kafkaClusterConfig.getTopic();
-
- logger.info(String.format("topic: %s, partitionId: %d, try to find closest offset with timestamp: %d between offset {%d, %d}", topic, partitionId, timestamp, firstAndLast.getFirst(), firstAndLast.getSecond()));
- final long result = binarySearch(kafkaClusterConfig, partitionId, firstAndLast.getFirst(), firstAndLast.getSecond(), timestamp, streamingParser);
- logger.info(String.format("topic: %s, partitionId: %d, found offset: %d", topic, partitionId, result));
- return result;
- }
-
- public static Pair<Long, Long> getFirstAndLastOffset(KafkaClusterConfig kafkaClusterConfig, int partitionId) {
- final String topic = kafkaClusterConfig.getTopic();
- final Broker leadBroker = Preconditions.checkNotNull(getLeadBroker(kafkaClusterConfig, partitionId), "unable to find leadBroker with config:" + kafkaClusterConfig + " partitionId:" + partitionId);
- final long earliestOffset = KafkaRequester.getLastOffset(topic, partitionId, OffsetRequest.EarliestTime(), leadBroker, kafkaClusterConfig);
- final long latestOffset = KafkaRequester.getLastOffset(topic, partitionId, OffsetRequest.LatestTime(), leadBroker, kafkaClusterConfig) - 1;
- return Pair.newPair(earliestOffset, latestOffset);
- }
-
- private static long binarySearch(KafkaClusterConfig kafkaClusterConfig, int partitionId, long startOffset, long endOffset, long targetTimestamp, StreamingParser streamingParser) {
- Map<Long, Long> cache = Maps.newHashMap();
-
- while (startOffset < endOffset) {
- long midOffset = startOffset + ((endOffset - startOffset) >> 1);
- long startTimestamp = getDataTimestamp(kafkaClusterConfig, partitionId, startOffset, streamingParser, cache);
- long endTimestamp = getDataTimestamp(kafkaClusterConfig, partitionId, endOffset, streamingParser, cache);
- long midTimestamp = getDataTimestamp(kafkaClusterConfig, partitionId, midOffset, streamingParser, cache);
- // hard to ensure these 2 conditions
- // Preconditions.checkArgument(startTimestamp <= midTimestamp);
- // Preconditions.checkArgument(midTimestamp <= endTimestamp);
- if (startTimestamp >= targetTimestamp) {
- return startOffset;
- }
- if (endTimestamp <= targetTimestamp) {
- return endOffset;
- }
- if (targetTimestamp == midTimestamp) {
- return midOffset;
- } else if (targetTimestamp < midTimestamp) {
- endOffset = midOffset - 1;
- continue;
- } else {
- startOffset = midOffset + 1;
- continue;
- }
- }
- return startOffset;
- }
-
- private static long getDataTimestamp(KafkaClusterConfig kafkaClusterConfig, int partitionId, long offset, StreamingParser streamingParser, Map<Long, Long> cache) {
- if (cache.containsKey(offset)) {
- return cache.get(offset);
- } else {
- long t = getDataTimestamp(kafkaClusterConfig, partitionId, offset, streamingParser);
- cache.put(offset, t);
- return t;
- }
- }
-
- public static long getDataTimestamp(KafkaClusterConfig kafkaClusterConfig, int partitionId, long offset, StreamingParser streamingParser) {
- final String topic = kafkaClusterConfig.getTopic();
- final MessageAndOffset messageAndOffset = getKafkaMessage(kafkaClusterConfig, partitionId, offset);
- final ByteBuffer payload = messageAndOffset.message().payload();
- byte[] bytes = new byte[payload.limit()];
- payload.get(bytes);
- final StreamingMessage streamingMessage = streamingParser.parse(messageAndOffset.message().payload());
- streamingMessage.setOffset(messageAndOffset.offset());
- logger.debug(String.format("The timestamp of topic: %s, partitionId: %d, offset: %d is: %d", topic, partitionId, offset, streamingMessage.getTimestamp()));
- return streamingMessage.getTimestamp();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/test/java/TimedJsonStreamParserTest.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/test/java/TimedJsonStreamParserTest.java b/source-kafka/src/test/java/TimedJsonStreamParserTest.java
index fb33059..5a52b61 100644
--- a/source-kafka/src/test/java/TimedJsonStreamParserTest.java
+++ b/source-kafka/src/test/java/TimedJsonStreamParserTest.java
@@ -20,7 +20,8 @@ import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.*;
+import java.io.File;
+import java.io.IOException;
import java.nio.ByteBuffer;
import com.fasterxml.jackson.databind.type.MapType;
@@ -32,7 +33,6 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.kafka.TimedJsonStreamParser;
-
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 1d33071..23e7239 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -41,10 +41,6 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-mr</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-streaming</artifactId>
- </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
deleted file mode 100644
index 9adaf24..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.storage.hbase.steps;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.inmemcubing.CompoundCuboidWriter;
-import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CuboidStatsUtil;
-import org.apache.kylin.engine.streaming.IStreamingOutput;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.IBuildable;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class HBaseStreamingOutput implements IStreamingOutput {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseStreamingOutput.class);
-
- @Override
- public ICuboidWriter getCuboidWriter(IBuildable buildable) {
- try {
- CubeSegment cubeSegment = (CubeSegment) buildable;
-
- final HTableInterface hTable;
- hTable = createHTable(cubeSegment);
- List<ICuboidWriter> cuboidWriters = Lists.newArrayList();
- cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
- cuboidWriters.add(new SequenceFileCuboidWriter(cubeSegment.getCubeDesc(), cubeSegment));
- return new CompoundCuboidWriter(cuboidWriters);
- } catch (IOException e) {
- throw new RuntimeException("failed to get ICuboidWriter", e);
- }
- }
-
- @Override
- public void output(IBuildable buildable, Map<Long, HyperLogLogPlusCounter> samplingResult) {
- try {
- CubeSegment cubeSegment = (CubeSegment) buildable;
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- final Configuration conf = HadoopUtil.getCurrentConfiguration();
- final Path outputPath = new Path("file://" + BatchConstants.CFG_STATISTICS_LOCAL_DIR + UUID.randomUUID().toString());
- CuboidStatsUtil.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
- FSDataInputStream inputStream = null;
- try {
- inputStream = FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME));
- ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), inputStream, System.currentTimeMillis());
- } finally {
- IOUtils.closeQuietly(inputStream);
- FileSystem.getLocal(conf).delete(outputPath, true);
- }
- } catch (IOException e) {
- throw new RuntimeException("failed to write sampling result", e);
- }
- }
-
- private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException {
- final String hTableName = cubeSegment.getStorageLocationIdentifier();
- CubeHTableUtil.createHTable(cubeSegment, null);
- final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
- logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!");
- return hTable;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index f4fb308..9cb135a 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -35,8 +35,8 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.streaming.StreamingConfig;
-import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
+import org.apache.kylin.metadata.streaming.StreamingManager;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.exception.PersistentException;
[05/50] [abbrv] kylin git commit: KYLIN-1726 minor update in kylin.sh
Posted by li...@apache.org.
KYLIN-1726 minor update in kylin.sh
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7847f51c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7847f51c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7847f51c
Branch: refs/heads/master-cdh5.7
Commit: 7847f51cac4d8fe2be494ed20bc3616a17ad8757
Parents: ed643e6
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 10 15:19:09 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 10 15:19:09 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 2 +-
.../apache/kylin/provision/BuildCubeWithStream.java | 15 +++++++--------
2 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7847f51c/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 039be9f..da53d3d 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -40,7 +40,7 @@ function retrieveDependency() {
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
- #export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${kafka_dependency}
+ #export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
}
# start command
http://git-wip-us.apache.org/repos/asf/kylin/blob/7847f51c/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index a47fcde..65dd151 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -276,11 +276,6 @@ public class BuildCubeWithStream {
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
- public static void afterClass() throws Exception {
- cleanupOldStorage();
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
public void after() {
kafkaServer.stop();
DefaultScheduler.destroyInstance();
@@ -301,6 +296,11 @@ public class BuildCubeWithStream {
}
}
+ public void cleanup() throws Exception {
+ cleanupOldStorage();
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
private static void cleanupOldStorage() throws Exception {
String[] args = { "--delete", "true" };
StorageCleanupJob cli = new StorageCleanupJob();
@@ -314,16 +314,15 @@ public class BuildCubeWithStream {
buildCubeWithStream = new BuildCubeWithStream();
buildCubeWithStream.before();
buildCubeWithStream.build();
+ logger.info("Build is done");
+ buildCubeWithStream.cleanup();
logger.info("Going to exit");
- System.exit(0);
} catch (Throwable e) {
logger.error("error", e);
- System.exit(1);
} finally {
if (buildCubeWithStream != null) {
buildCubeWithStream.after();
}
- afterClass();
}
}
[44/50] [abbrv] kylin git commit: KYLIN-1839,
support kylin lib in HDFS
Posted by li...@apache.org.
KYLIN-1839, support kylin lib in HDFS
Signed-off-by: terry <hz...@corp.netease.com>
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4473d710
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4473d710
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4473d710
Branch: refs/heads/master-cdh5.7
Commit: 4473d71011cc0e652eccf4f80269828caa5d3c73
Parents: d28835f
Author: terry <hz...@corp.netease.com>
Authored: Tue Oct 11 17:33:45 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Oct 19 07:30:13 2016 +0800
----------------------------------------------------------------------
.../engine/mr/common/AbstractHadoopJob.java | 29 +++++++++++---------
1 file changed, 16 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4473d710/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a138eec..bbb1711 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -252,14 +253,10 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
if (!StringUtils.isBlank(mrLibDir)) {
- File dirFileMRLIB = new File(mrLibDir);
- if (dirFileMRLIB.exists()) {
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
+ if(kylinDependency.length() > 0) {
+ kylinDependency.append(",");
+ }
kylinDependency.append(mrLibDir);
- } else {
- logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!");
- }
}
setJobTmpJarsAndFiles(job, kylinDependency.toString());
@@ -300,21 +297,27 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
try {
Configuration jobConf = job.getConfiguration();
FileSystem fs = FileSystem.getLocal(jobConf);
+ FileSystem hdfs = FileSystem.get(jobConf);
StringBuilder jarList = new StringBuilder();
StringBuilder fileList = new StringBuilder();
for (String fileName : fNameList) {
Path p = new Path(fileName);
- if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, fileName);
+ FileSystem current = (fileName.startsWith(HdfsConstants.HDFS_URI_SCHEME) ? hdfs : fs);
+ if(!current.exists(p)) {
+ logger.warn("The directory '" + fileName + "for kylin dependency does not exist!!!");
+ continue;
+ }
+ if (current.getFileStatus(p).isDirectory()) {
+ appendTmpDir(job, current, fileName);
continue;
}
StringBuilder list = (p.getName().endsWith(".jar")) ? jarList : fileList;
if (list.length() > 0)
list.append(",");
- list.append(fs.getFileStatus(p).getPath().toString());
+ list.append(current.getFileStatus(p).getPath());
}
appendTmpFiles(fileList.toString(), jobConf);
@@ -324,13 +327,12 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
- private void appendTmpDir(Job job, String tmpDir) {
+ private void appendTmpDir(Job job, FileSystem fs, String tmpDir) {
if (StringUtils.isBlank(tmpDir))
return;
try {
Configuration jobConf = job.getConfiguration();
- FileSystem fs = FileSystem.getLocal(jobConf);
FileStatus[] fList = fs.listStatus(new Path(tmpDir));
StringBuilder jarList = new StringBuilder();
@@ -339,7 +341,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
for (FileStatus file : fList) {
Path p = file.getPath();
if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, p.toString());
+ appendTmpDir(job, fs, p.toString());
continue;
}
@@ -622,3 +624,4 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
+
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8391565
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8391565
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8391565
Branch: refs/heads/master-cdh5.7
Commit: b8391565e7855ea545c0a4fac72e0b4b5c06c872
Parents: db09f5f
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Oct 19 10:48:12 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../kylin/provision/BuildCubeWithEngine.java | 13 ++-
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 35 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../kylin/storage/hbase/HBaseStorage.java | 3 +-
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +-
.../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
.../cube/v1/SerializedHBaseTupleIterator.java | 4 +-
.../observer/AggregateRegionObserver.java | 4 +-
.../observer/AggregationScanner.java | 14 ++-
.../observer/ObserverAggregationCache.java | 10 ++-
.../coprocessor/observer/ObserverEnabler.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 23 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 22 ++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../observer/AggregateRegionObserverTest.java | 26 ++----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +-
48 files changed, 403 insertions(+), 457 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index f6c8801..c46c0cc 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -55,6 +54,7 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
@@ -425,10 +425,10 @@ public class BuildCubeWithEngine {
}
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
@@ -454,5 +454,4 @@ public class BuildCubeWithEngine {
}
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c7538e4..3b666dd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 38f299e..bfb5fe4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -37,6 +37,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index ab18029..d55edc3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -57,11 +58,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 99e54b9..6a2e9f5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,9 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -410,33 +408,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- IOUtils.closeQuietly(table);
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 2803125..a1b4079 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -46,11 +46,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
@@ -163,14 +163,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -196,14 +195,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -215,12 +213,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 3fd6426..169510a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,14 +31,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- HConnection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,7 +273,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -288,7 +286,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -310,7 +307,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -318,7 +315,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -337,7 +334,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -363,7 +360,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -371,8 +368,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index f4dfd2b..3d82105 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -18,7 +18,6 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.base.Preconditions;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
@@ -36,6 +35,8 @@ import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings("unused")
//used by reflection
public class HBaseStorage implements IStorage {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 8ac3832..982a044 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -25,11 +25,12 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
@@ -70,7 +71,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
protected final List<RowValueDecoder> rowValueDecoders;
private final StorageContext context;
private final String tableName;
- private final HTableInterface table;
+ private final Table table;
protected CubeTupleConverter tupleConverter;
protected final Iterator<HBaseKeyRange> rangeIterator;
@@ -88,7 +89,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private int advMeasureRowsRemaining;
private int advMeasureRowIndex;
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
+ public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, Connection conn, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -108,7 +109,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
this.rangeIterator = keyRanges.iterator();
try {
- this.table = conn.getTable(tableName);
+ this.table = conn.getTable(TableName.valueOf(tableName));
} catch (Throwable t) {
throw new StorageException("Error when open connection to table " + tableName, t);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index ff729f4..1944327 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
@@ -46,10 +46,10 @@ import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -152,7 +152,7 @@ public class CubeStorageQuery implements IStorageQuery {
setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
setLimit(filter, context);
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ Connection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 6342c5c..0ade920 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -50,7 +51,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -60,7 +61,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -94,4 +95,9 @@ public class RegionScannerAdapter implements RegionScanner {
return Long.MAX_VALUE;
}
+ @Override
+ public int getBatch() {
+ return -1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index e8dd5b9..d033c77 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -57,7 +57,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private int scanCount;
private ITuple next;
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
+ public SerializedHBaseTupleIterator(Connection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7139ca7..7e25e4c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
+ Region region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index a900ea1..d64f48f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
@@ -116,8 +117,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.next(result, scannerContext);
}
@Override
@@ -126,8 +127,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.nextRaw(result, scannerContext);
}
@Override
@@ -160,6 +161,11 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
+ @Override
+ public int getBatch() {
+ return outerScanner.getBatch();
+ }
+
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 8404262..331e34d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -112,7 +113,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -122,7 +123,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -161,6 +162,11 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
+
+ @Override
+ public int getBatch() {
+ return innerScanner.getBatch();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index 394b3e2..9fd33f5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -23,9 +23,9 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -60,7 +60,7 @@ public class ObserverEnabler {
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
+ Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, Table table, Scan scan) throws IOException {
if (context.isCoprocessorEnabled() == false) {
return table.getScanner(scan);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 4f538ae..dd468cf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -50,10 +51,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -169,7 +170,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 3cefc5f..a52af90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -154,8 +155,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 506778c..3d8e494 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -179,7 +179,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Kylin Query-%s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 2814ad6..feb4842 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 21f1c00..0f214eb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -29,9 +29,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -100,19 +101,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index d5b36df..6587d4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,7 +117,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
[19/50] [abbrv] kylin git commit: KYLIN-2085 Add test cases
Posted by li...@apache.org.
KYLIN-2085 Add test cases
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96935fe9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96935fe9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96935fe9
Branch: refs/heads/master-cdh5.7
Commit: 96935fe99996cd02efe0b8d11d25f2f67a83ad17
Parents: 8fbfc58
Author: lidongsjtu <li...@apache.org>
Authored: Wed Oct 12 17:21:24 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Oct 12 17:29:00 2016 +0800
----------------------------------------------------------------------
.../src/test/resources/query/sql/query100.sql | 20 ++++++++++++++++++++
.../resources/query/sql_dynamic/query04.dat | 2 ++
.../resources/query/sql_dynamic/query04.sql | 20 ++++++++++++++++++++
3 files changed, 42 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/96935fe9/kylin-it/src/test/resources/query/sql/query100.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query100.sql b/kylin-it/src/test/resources/query/sql/query100.sql
new file mode 100644
index 0000000..a2b0d1b
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query100.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as cnt from test_kylin_fact
+where lstg_format_name>='AAAA' and 'BBBB'>=lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/96935fe9/kylin-it/src/test/resources/query/sql_dynamic/query04.dat
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_dynamic/query04.dat b/kylin-it/src/test/resources/query/sql_dynamic/query04.dat
new file mode 100644
index 0000000..e8506ab
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_dynamic/query04.dat
@@ -0,0 +1,2 @@
+AAAA
+BBBB
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/96935fe9/kylin-it/src/test/resources/query/sql_dynamic/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_dynamic/query04.sql b/kylin-it/src/test/resources/query/sql_dynamic/query04.sql
new file mode 100644
index 0000000..135cf69
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_dynamic/query04.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as cnt from test_kylin_fact
+where lstg_format_name>=? and ?>=lstg_format_name
\ No newline at end of file
[42/50] [abbrv] kylin git commit: Ignore ITKylinQueryTest as it will
be invoked by sub-classes
Posted by li...@apache.org.
Ignore ITKylinQueryTest as it will be invoked by sub-classes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c84892f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c84892f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c84892f5
Branch: refs/heads/master-cdh5.7
Commit: c84892f527a6a16904d3cb801748e35248b4d90a
Parents: 03118bd
Author: shaofengshi <sh...@apache.org>
Authored: Tue Oct 18 15:14:49 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Oct 18 15:17:01 2016 +0800
----------------------------------------------------------------------
kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c84892f5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index ffd85e3..2ec5324 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -45,6 +45,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
[40/50] [abbrv] kylin git commit: KYLIN-2091 bug fix
Posted by li...@apache.org.
KYLIN-2091 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c13ab1c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c13ab1c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c13ab1c3
Branch: refs/heads/master-cdh5.7
Commit: c13ab1c324bf6a56e75365e0e1ab5ecd48fd048a
Parents: 07e81fd
Author: shaofengshi <sh...@apache.org>
Authored: Tue Oct 18 10:43:13 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Oct 18 10:43:13 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/CubeSegment.java | 12 ------------
.../apache/kylin/source/kafka/job/SeekOffsetStep.java | 2 ++
2 files changed, 2 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index eb5b389..fdf1fb0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,12 +564,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
public void setSourcePartitionOffsetEnd(Map<Integer, Long> sourcePartitionOffsetEnd) {
this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
- long total = 0;
- for (Long x : sourcePartitionOffsetEnd.values()) {
- total += x;
- }
-
- this.sourceOffsetEnd = total;
}
public Map<Integer, Long> getSourcePartitionOffsetStart() {
@@ -578,11 +572,5 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
public void setSourcePartitionOffsetStart(Map<Integer, Long> sourcePartitionOffsetStart) {
this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
- long total = 0;
- for (Long x : sourcePartitionOffsetStart.values()) {
- total += x;
- }
-
- this.sourceOffsetStart = total;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
index 2a3dbb5..a26f39d 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
@@ -116,6 +116,8 @@ public class SeekOffsetStep extends AbstractExecutable {
}
if (totalEndOffset > totalStartOffset) {
+ segment.setSourceOffsetStart(totalStartOffset);
+ segment.setSourceOffsetEnd(totalEndOffset);
segment.setSourcePartitionOffsetStart(startOffsets);
segment.setSourcePartitionOffsetEnd(endOffsets);
segment.setName(CubeSegment.makeSegmentName(0, 0, totalStartOffset, totalEndOffset));
[13/50] [abbrv] kylin git commit: fix CI
Posted by li...@apache.org.
fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fad9e818
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fad9e818
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fad9e818
Branch: refs/heads/master-cdh5.7
Commit: fad9e818fe965e15dd086bb2c33d63784b347e44
Parents: 0c7e268
Author: shaofengshi <sh...@apache.org>
Authored: Tue Oct 11 17:10:16 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Oct 11 17:10:32 2016 +0800
----------------------------------------------------------------------
.../test/java/org/apache/kylin/provision/BuildCubeWithStream.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fad9e818/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 65dd151..d2e46a5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -317,8 +317,10 @@ public class BuildCubeWithStream {
logger.info("Build is done");
buildCubeWithStream.cleanup();
logger.info("Going to exit");
+ System.exit(0);
} catch (Throwable e) {
logger.error("error", e);
+ System.exit(1);
} finally {
if (buildCubeWithStream != null) {
buildCubeWithStream.after();
[17/50] [abbrv] kylin git commit: KYLIN-2068: add optional comment
column when sync hive table columns
Posted by li...@apache.org.
KYLIN-2068: add optional comment column when sync hive table columns
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58afe010
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58afe010
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58afe010
Branch: refs/heads/master-cdh5.7
Commit: 58afe0103098dd0845cb50a0751e3305a973f834
Parents: 460536c
Author: Yiming Liu <li...@gmail.com>
Authored: Mon Oct 10 23:49:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Oct 12 09:25:24 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/ColumnDesc.java | 13 ++++++
examples/sample_cube/create_sample_tables.sql | 49 +++++++++++---------
.../source/hive/HiveSourceTableLoader.java | 1 +
webapp/app/js/model/tableConfig.js | 3 +-
webapp/app/partials/tables/table_detail.html | 4 ++
5 files changed, 46 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 257cb3b..d131b92 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
+import com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
@@ -51,6 +52,9 @@ public class ColumnDesc implements Serializable {
private String name;
@JsonProperty("datatype")
private String datatype;
+ @JsonProperty("comment")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String comment;
// parsed from data type
private DataType type;
@@ -109,6 +113,14 @@ public class ColumnDesc implements Serializable {
this.table = table;
}
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
public DataType getType() {
return type;
}
@@ -195,6 +207,7 @@ public class ColumnDesc implements Serializable {
"id='" + id + '\'' +
", name='" + name + '\'' +
", datatype='" + datatype + '\'' +
+ ", comment='" + comment + '\'' +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index 8399ee8..fa5e25b 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -20,11 +20,11 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
CREATE TABLE DEFAULT.KYLIN_CAL_DT
(
-CAL_DT date
-,YEAR_BEG_DT date
-,QTR_BEG_DT date
-,MONTH_BEG_DT date
-,WEEK_BEG_DT date
+CAL_DT date COMMENT 'Date, PK'
+,YEAR_BEG_DT date COMMENT 'YEAR Begin Date'
+,QTR_BEG_DT date COMMENT 'Quarter Begin Date'
+,MONTH_BEG_DT date COMMENT 'Month Begin Date'
+,WEEK_BEG_DT date COMMENT 'Week Begin Date'
,AGE_FOR_YEAR_ID smallint
,AGE_FOR_QTR_ID smallint
,AGE_FOR_MONTH_ID smallint
@@ -121,6 +121,7 @@ CAL_DT date
,UPD_DATE string
,UPD_USER string
)
+COMMENT 'Date Dimension Table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
@@ -128,28 +129,28 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
(
-LEAF_CATEG_ID bigint
+LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
,LEAF_CATEG_NAME string
-,SITE_ID int
+,SITE_ID int COMMENT 'Site ID, PK'
,CATEG_BUSN_MGR string
,CATEG_BUSN_UNIT string
,REGN_CATEG string
-,USER_DEFINED_FIELD1 string
-,USER_DEFINED_FIELD3 string
+,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
+,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
,CRE_DATE string
-,UPD_DATE string
+,UPD_DATE string COMMENT 'Last Updated Date'
,CRE_USER string
-,UPD_USER string
+,UPD_USER string COMMENT 'Last Updated User'
,META_CATEG_ID decimal
-,META_CATEG_NAME string
+,META_CATEG_NAME string COMMENT 'Level1 Category'
,CATEG_LVL2_ID decimal
,CATEG_LVL3_ID decimal
,CATEG_LVL4_ID decimal
,CATEG_LVL5_ID decimal
,CATEG_LVL6_ID decimal
,CATEG_LVL7_ID decimal
-,CATEG_LVL2_NAME string
-,CATEG_LVL3_NAME string
+,CATEG_LVL2_NAME string COMMENT 'Level2 Category'
+,CATEG_LVL3_NAME string COMMENT 'Level3 Category'
,CATEG_LVL4_NAME string
,CATEG_LVL5_NAME string
,CATEG_LVL6_NAME string
@@ -165,6 +166,7 @@ LEAF_CATEG_ID bigint
,SRC_ID tinyint
,BSNS_VRTCL_NAME string
)
+COMMENT 'Detail category inforamtion, Dimension Table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
@@ -173,17 +175,18 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
CREATE TABLE DEFAULT.KYLIN_SALES
(
TRANS_ID bigint
-,PART_DT date
-,LSTG_FORMAT_NAME string
-,LEAF_CATEG_ID bigint
-,LSTG_SITE_ID int
+,PART_DT date COMMENT 'Order Date'
+,LSTG_FORMAT_NAME string COMMENT 'Order Transaction Type'
+,LEAF_CATEG_ID bigint COMMENT 'Category ID'
+,LSTG_SITE_ID int COMMENT 'Site ID'
,SLR_SEGMENT_CD smallint
-,PRICE decimal(19,4)
-,ITEM_COUNT bigint
-,SELLER_ID bigint
-,USER_ID string
-,REGION string
+,PRICE decimal(19,4) COMMENT 'Order Price'
+,ITEM_COUNT bigint COMMENT 'Number of Purchased Goods'
+,SELLER_ID bigint COMMENT 'Seller ID'
+,USER_ID string COMMENT 'USER ID'
+,REGION string COMMENT 'REGION'
)
+COMMENT 'Sales order table, fact table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 8b98e7b..ec9aedb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -133,6 +133,7 @@ public class HiveSourceTableLoader {
cdesc.setDatatype(field.getType());
}
cdesc.setId(String.valueOf(i + 1));
+ cdesc.setComment(field.getComment());
columns.add(cdesc);
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/webapp/app/js/model/tableConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/tableConfig.js b/webapp/app/js/model/tableConfig.js
index 3dd7b25..2198018 100644
--- a/webapp/app/js/model/tableConfig.js
+++ b/webapp/app/js/model/tableConfig.js
@@ -21,7 +21,8 @@ KylinApp.constant('tableConfig', {
{attr: 'id', name: 'ID'},
{attr: 'name', name: 'Name'},
{attr: 'datatype', name: 'Data Type'},
- {attr: 'cardinality', name: 'Cardinality'}
+ {attr: 'cardinality', name: 'Cardinality'},
+ {attr: 'comment', name: 'Comment'}
],
dataTypes:["tinyint","smallint","int","bigint","float","double","decimal","timestamp","date","string","varchar(256)","char","boolean","binary"]
http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/webapp/app/partials/tables/table_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/table_detail.html b/webapp/app/partials/tables/table_detail.html
index a4548a1..5bae00f 100644
--- a/webapp/app/partials/tables/table_detail.html
+++ b/webapp/app/partials/tables/table_detail.html
@@ -136,6 +136,10 @@
<!--{{ tableModel.selectedSrcTable.cardinality[column.name]}}-->
{{column.cardinality}}
</td>
+ <td
+ style="{{(tableModel.selectedSrcTable.selectedSrcColumn.id == column.id)? 'background-color:#EBF9FE':''}}">
+ {{ column.comment}}
+ </td>
</tr>
</table>
</div>
[11/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and
data
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 13c7d85..dffa43b 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,10000 +1,10000 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,10000349
-0,2012-08-28,Others,175750,0,13,23.8563,0,10000927
-0,2012-02-16,ABIN,148324,15,13,88.3418,0,10000005
-0,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,10000209
-0,2012-10-22,Others,140746,100,11,83.4540,0,10000154
-0,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,10000372
-0,2013-04-04,Others,963,0,13,88.5907,0,10000648
-0,2012-04-11,Others,15687,0,15,88.1940,0,10000866
-0,2013-01-19,ABIN,60606,3,13,77.9727,0,10000936
-0,2012-04-30,FP-non GTC,106246,0,14,52.2950,0,10000047
-0,2013-02-03,Auction,45333,0,16,56.3584,0,10000470
-0,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,10000203
-0,2013-07-30,Others,24760,0,16,25.0770,0,10000382
-0,2012-04-17,Auction,31519,0,16,72.7384,0,10000069
-0,2013-04-30,FP-GTC,51582,0,15,75.8200,0,10000085
-0,2013-03-23,FP-GTC,16509,0,15,8.6653,0,10000559
-0,2013-04-01,ABIN,175750,3,16,64.2802,0,10000507
-0,2013-07-12,ABIN,50677,0,-99,24.5987,0,10000938
-0,2012-04-23,FP-GTC,45333,0,5,72.6553,0,10000863
-0,2012-12-19,ABIN,3838,0,12,87.4142,0,10000612
-0,2012-08-15,FP-GTC,62179,0,16,67.4238,0,10000839
-0,2013-08-22,Auction,1357,0,5,18.3600,0,10000303
-0,2013-08-23,ABIN,31519,0,5,67.6344,0,10000877
-0,2013-09-12,FP-GTC,223,0,14,71.7898,0,10000848
-0,2012-03-14,FP-GTC,132939,0,15,26.1496,0,10000815
-0,2013-07-13,Others,67703,3,16,83.9661,0,10000858
-0,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,10000387
-0,2012-08-30,Auction,62179,0,11,21.0496,0,10000393
-0,2013-01-10,FP-GTC,48027,0,13,23.4142,0,10000757
-0,2013-01-25,ABIN,145970,0,5,26.7842,0,10000402
-0,2012-07-05,Auction,48027,0,-99,46.7320,0,10000972
-0,2012-11-07,FP-GTC,20213,0,-99,19.7270,0,10000873
-0,2013-06-20,ABIN,100847,0,5,76.2783,0,10000409
-0,2012-10-08,ABIN,26262,0,5,48.7447,0,10000029
-0,2013-04-30,ABIN,87118,0,15,63.2187,0,10000529
-0,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,10000329
-0,2012-05-13,Others,164261,0,13,84.1246,0,10000688
-0,2013-08-02,ABIN,31519,0,5,37.1504,0,10000905
-0,2012-04-03,Auction,164261,0,11,92.0974,0,10000341
-0,2012-09-02,FP-GTC,31519,0,14,87.7829,0,10000460
-0,2013-10-05,ABIN,13987,0,16,74.0719,0,10000381
-0,2012-01-13,FP-GTC,48027,0,15,71.2637,0,10000915
-0,2013-03-01,Others,13836,0,14,16.7288,0,10000953
-0,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,10000277
-0,2012-02-07,Auction,45333,0,5,64.9770,0,10000040
-0,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,10000500
-0,2013-10-13,Auction,31519,0,5,79.3053,0,10000816
-0,2012-06-11,ABIN,4943,0,5,11.6942,0,10000696
-0,2012-10-18,ABIN,80053,0,-99,54.0933,0,10000604
-0,2012-11-03,Auction,20886,0,5,9.8258,0,10000764
-0,2012-01-15,Auction,44079,0,14,13.0371,0,10000343
-0,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,10000724
-0,2012-02-16,FP-GTC,314,0,13,49.1825,0,10000334
-0,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,10000603
-0,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,10000248
-0,2012-07-24,FP-GTC,150265,15,16,57.5645,0,10000653
-0,2012-10-06,Auction,145970,0,5,44.3091,0,10000082
-0,2013-06-14,ABIN,80053,0,5,20.2388,0,10000632
-0,2013-10-10,FP-non GTC,15868,0,11,28.6290,0,10000523
-0,2012-09-30,ABIN,31673,0,12,75.8658,0,10000719
-0,2012-04-02,FP-GTC,63861,0,5,25.8748,0,10000311
-0,2012-03-16,Others,314,0,-99,9.2733,0,10000409
-0,2012-09-30,FP-non GTC,60340,0,14,74.6610,0,10000790
-0,2012-03-20,Others,87118,0,5,79.3781,0,10000384
-0,2012-09-12,FP-GTC,1120,3,5,61.4408,0,10000645
-0,2013-12-10,FP-non GTC,20886,0,11,14.8140,0,10000943
-0,2012-12-05,FP-GTC,106246,0,12,24.9380,0,10000858
-0,2012-08-21,Auction,43479,0,12,22.2008,0,10000310
-0,2012-11-30,FP-GTC,175750,0,15,35.1161,0,10000440
-0,2013-10-10,Others,95672,0,11,82.3718,0,10000128
-0,2012-10-17,Auction,80053,0,15,9.8166,0,10000105
-0,2012-03-16,FP-non GTC,57784,0,5,66.2369,0,10000439
-0,2012-11-14,ABIN,65,0,11,17.5513,0,10000855
-0,2013-06-24,FP-GTC,16509,0,15,53.9195,0,10000130
-0,2012-08-13,FP-non GTC,65,0,5,69.8447,0,10000548
-0,2012-01-03,Others,38238,0,11,90.3722,0,10000739
-0,2013-11-24,FP-non GTC,175750,3,13,47.3441,0,10000291
-0,2013-11-08,Auction,106246,0,5,38.6729,0,10000310
-0,2013-04-07,Others,73506,0,-99,69.8999,0,10000235
-0,2012-10-15,FP-GTC,314,0,13,44.1598,0,10000265
-0,2012-01-20,FP-GTC,80287,0,-99,85.7263,0,10000704
-0,2012-08-17,ABIN,32876,0,16,21.2934,0,10000163
-0,2013-07-09,Auction,37831,0,15,38.2213,0,10000557
-0,2013-09-21,Others,45333,0,13,25.5384,0,10000184
-0,2013-10-28,FP-non GTC,11554,0,16,89.9654,0,10000940
-0,2013-04-05,Auction,11554,0,14,41.8676,0,10000384
-0,2013-03-25,FP-GTC,43398,0,14,75.0110,0,10000697
-0,2012-05-03,ABIN,33038,15,12,64.6735,0,10000147
-0,2012-10-13,ABIN,95672,3,14,3.2128,0,10000547
-0,2012-09-11,FP-GTC,37831,0,16,60.3228,0,10000899
-0,2012-08-31,Auction,314,0,5,13.2480,0,10000350
-0,2012-12-25,Others,165888,0,16,60.3492,0,10000367
-0,2012-12-26,FP-non GTC,50677,0,5,58.6078,0,10000523
-0,2012-05-24,FP-GTC,145970,0,5,33.1342,0,10000926
-0,2012-06-27,Auction,174106,3,13,55.3039,0,10000430
-0,2012-01-02,ABIN,87118,0,14,48.2433,0,10000731
-0,2013-12-14,FP-GTC,50508,0,5,51.5822,0,10000977
-0,2012-12-30,Auction,63889,0,12,3.8393,0,10000957
-0,2013-10-26,FP-non GTC,24541,0,5,23.6812,0,10000350
-0,2013-12-29,ABIN,38238,0,15,99.4389,0,10000605
-0,2012-05-26,FP-GTC,100847,0,15,11.0672,0,10000291
-0,2012-12-18,FP-non GTC,164261,0,11,54.9352,0,10000509
-0,2013-03-12,ABIN,95672,0,15,85.7152,0,10000742
-0,2013-02-22,ABIN,43398,0,11,65.1970,0,10000294
-0,2012-02-20,Others,175750,0,14,63.2187,0,10000364
-0,2013-08-06,Others,32876,0,13,50.8111,0,10000542
-0,2013-06-27,Others,13987,0,13,2.3165,0,10000979
-0,2012-07-17,Others,13987,0,16,25.2136,0,10000744
-0,2013-10-12,FP-non GTC,16509,0,5,15.3986,0,10000652
-0,2012-12-07,Others,158798,0,5,1.6111,0,10000149
-0,2013-02-09,Auction,32876,0,5,37.3237,0,10000467
-0,2012-10-26,Others,44079,0,12,18.1870,0,10000420
-0,2012-04-10,ABIN,132939,0,13,88.5950,0,10000103
-0,2013-04-08,FP-non GTC,159184,0,5,92.4421,0,10000933
-0,2012-05-30,Others,80287,0,14,68.1663,0,10000195
-0,2012-02-23,FP-GTC,88750,0,11,19.4075,0,10000038
-0,2013-10-25,ABIN,11554,0,11,27.2512,0,10000228
-0,2012-12-02,FP-GTC,41940,0,14,49.5504,0,10000927
-0,2013-09-16,FP-GTC,314,0,-99,3.4785,0,10000304
-0,2013-11-01,Others,31519,3,15,46.1783,0,10000771
-0,2012-10-21,FP-non GTC,31519,0,15,1.7696,0,10000507
-0,2013-08-25,Auction,95173,0,13,89.2083,0,10000868
-0,2013-11-09,FP-non GTC,150047,3,12,58.2634,0,10000497
-0,2012-02-04,FP-GTC,175750,0,11,41.3850,0,10000947
-0,2012-10-17,ABIN,26249,0,14,10.5789,0,10000965
-0,2012-03-16,FP-non GTC,963,0,5,17.8949,0,10000908
-0,2012-08-06,Others,94847,0,13,96.0332,0,10000595
-0,2013-12-23,FP-non GTC,24760,0,16,90.7096,0,10000700
-0,2013-03-29,FP-GTC,9426,3,13,50.9023,0,10000090
-0,2013-03-25,Others,20886,0,-99,35.0414,0,10000171
-0,2013-12-10,Auction,95672,0,15,29.8872,0,10000216
-0,2012-03-14,Others,12688,0,14,10.4067,0,10000626
-0,2013-11-29,FP-non GTC,31519,0,13,89.2334,0,10000199
-0,2013-04-29,ABIN,31519,0,-99,99.2460,0,10000565
-0,2012-02-01,FP-GTC,161567,15,11,56.3989,0,10000651
-0,2012-03-17,FP-GTC,4943,0,16,0.4422,0,10000829
-0,2012-11-06,FP-GTC,156614,0,12,2.3454,0,10000370
-0,2012-12-03,Auction,155226,0,13,51.1354,0,10000867
-0,2012-09-09,FP-GTC,99985,0,12,9.1236,0,10000281
-0,2012-06-16,ABIN,99985,0,16,16.4070,0,10000143
-0,2012-05-06,FP-GTC,99985,0,14,6.4304,0,10000514
-0,2013-10-20,FP-GTC,36250,0,13,55.6895,0,10000390
-0,2012-10-11,ABIN,106246,0,15,30.6373,0,10000980
-0,2013-08-08,Others,103178,15,12,41.4904,0,10000751
-0,2012-09-05,Auction,314,0,16,3.2615,0,10000280
-0,2012-08-07,ABIN,48904,0,13,57.7688,0,10000245
-0,2013-07-05,ABIN,314,0,14,28.9444,0,10000888
-0,2012-09-10,FP-non GTC,95672,0,-99,54.0914,0,10000601
-0,2012-09-18,Auction,25147,0,12,74.4422,0,10000679
-0,2012-08-24,FP-non GTC,175750,3,16,45.8282,0,10000397
-0,2013-05-29,Others,26249,0,14,12.7681,0,10000967
-0,2012-11-17,FP-GTC,94847,0,14,74.4181,0,10000175
-0,2012-09-22,ABIN,175750,0,15,85.6879,0,10000453
-0,2013-09-17,ABIN,533,0,11,85.5838,0,10000642
-0,2013-03-28,Others,60340,0,12,72.6495,0,10000397
-0,2013-08-09,Others,216,0,15,88.6918,0,10000867
-0,2013-12-19,Others,43398,0,-99,88.0444,0,10000482
-0,2013-12-02,FP-non GTC,15115,0,-99,29.6027,0,10000538
-0,2012-11-23,Others,46575,0,5,49.3124,0,10000678
-0,2013-06-25,FP-non GTC,16145,0,15,7.4413,0,10000897
-0,2012-01-22,FP-GTC,164,0,14,71.8071,0,10000510
-0,2013-04-24,FP-GTC,1504,0,12,95.5706,0,10000554
-0,2012-03-05,FP-non GTC,12688,0,5,47.7646,0,10000175
-0,2013-02-15,FP-non GTC,16145,0,13,40.9511,0,10000647
-0,2013-01-18,Others,36250,0,13,77.3572,0,10000687
-0,2012-06-04,Others,15115,0,15,25.3400,0,10000493
-0,2013-10-13,FP-GTC,165888,0,-99,58.1861,0,10000753
-0,2012-05-21,Auction,145970,0,5,37.2504,0,10000292
-0,2012-04-26,Auction,11848,0,13,29.2954,0,10000804
-0,2013-09-26,FP-GTC,61323,0,13,84.7095,0,10000761
-0,2013-03-06,Auction,100847,0,-99,81.7980,0,10000593
-0,2012-08-08,ABIN,51582,0,15,15.9790,0,10000138
-0,2012-01-09,Others,1504,0,14,34.0508,0,10000954
-0,2013-10-20,ABIN,20886,0,12,36.9353,0,10000408
-0,2013-06-25,Auction,53064,0,11,99.7714,0,10000409
-0,2013-05-03,FP-GTC,63861,0,5,39.2974,0,10000668
-0,2013-01-26,FP-GTC,100847,0,13,59.5009,0,10000536
-0,2012-09-19,FP-GTC,164261,0,13,15.7583,0,10000231
-0,2013-11-16,FP-GTC,164261,0,16,83.6234,0,10000927
-0,2013-05-07,FP-non GTC,20485,0,5,52.9100,0,10000491
-0,2013-08-15,Others,32996,15,12,54.7122,0,10000411
-0,2013-10-31,FP-GTC,63861,0,14,0.3599,0,10000903
-0,2012-06-13,FP-non GTC,159184,0,11,49.9550,0,10000849
-0,2012-12-25,Auction,20213,0,-99,11.4752,0,10000583
-0,2013-12-15,Auction,759,0,13,33.6991,0,10000390
-0,2012-04-30,Others,87118,0,16,51.4630,0,10000572
-0,2012-07-14,Others,10866,0,11,36.9614,0,10000467
-0,2013-06-25,FP-non GTC,1357,0,12,17.5618,0,10000426
-0,2013-02-06,Auction,13836,0,12,74.3179,0,10000346
-0,2012-06-03,ABIN,60340,0,12,82.3722,0,10000531
-0,2013-07-23,FP-GTC,139973,0,-99,75.2548,0,10000385
-0,2013-07-06,Others,11554,0,14,40.1129,0,10000724
-0,2013-08-10,FP-non GTC,31519,0,-99,5.2432,0,10000356
-0,2013-07-31,Auction,31673,0,14,51.1089,0,10000598
-0,2013-12-05,Auction,314,0,13,19.9031,0,10000068
-0,2012-12-30,ABIN,50508,0,5,73.8659,0,10000482
-0,2012-09-22,Auction,165888,0,12,81.9826,0,10000832
-0,2012-07-16,Auction,223,0,5,76.0284,0,10000354
-0,2013-01-26,Others,20485,0,12,71.6623,0,10000552
-0,2012-11-07,FP-non GTC,57013,0,16,73.2931,0,10000801
-0,2012-01-26,FP-non GTC,175750,0,13,18.8970,0,10000649
-0,2012-06-09,FP-non GTC,60340,0,12,11.1614,0,10000830
-0,2012-06-20,Auction,57990,3,5,69.1183,0,10000557
-0,2013-01-01,FP-non GTC,45333,0,13,94.0586,0,10000253
-0,2013-07-26,FP-non GTC,139973,0,11,39.7239,0,10000441
-0,2012-09-15,FP-GTC,24541,0,15,0.3239,0,10000379
-0,2013-09-22,Auction,31519,3,12,80.0315,0,10000436
-0,2012-10-27,FP-GTC,223,0,15,39.3790,0,10000420
-0,2012-11-20,FP-non GTC,533,0,5,49.4951,0,10000390
-0,2013-12-07,FP-GTC,1349,0,15,93.3008,0,10000398
-0,2012-05-30,FP-non GTC,46575,0,12,66.7652,0,10000915
-0,2012-09-07,FP-non GTC,95672,0,14,29.5970,0,10000231
-0,2013-04-26,Auction,31673,0,-99,15.8347,0,10000260
-0,2012-09-11,FP-non GTC,50508,0,14,80.5444,0,10000263
-0,2013-09-03,FP-non GTC,95173,0,16,33.1487,0,10000493
-0,2012-10-03,FP-non GTC,44079,0,15,99.7408,0,10000828
-0,2012-12-26,ABIN,16145,0,14,40.7826,0,10000062
-0,2012-07-17,Auction,26249,0,15,28.6663,0,10000534
-0,2013-06-30,ABIN,63861,0,14,75.6832,0,10000944
-0,2012-01-09,ABIN,156614,0,11,98.4996,0,10000353
-0,2013-09-08,FP-GTC,156614,0,13,12.7473,0,10000063
-0,2013-09-23,Auction,41940,0,5,37.5254,0,10000383
-0,2012-11-11,Auction,145970,0,5,77.1361,0,10000420
-0,2013-08-26,Others,11848,0,16,28.2799,0,10000323
-0,2012-09-15,Others,11554,0,13,63.7232,0,10000729
-0,2012-01-09,FP-non GTC,25147,0,11,7.3009,0,10000745
-0,2012-08-16,Others,87118,0,15,47.8997,0,10000240
-0,2013-12-26,FP-non GTC,16145,0,12,51.0442,0,10000050
-0,2012-03-24,ABIN,48027,0,5,26.2083,0,10000600
-0,2012-08-22,ABIN,216,0,5,79.9268,0,10000706
-0,2013-11-03,FP-GTC,24541,0,12,83.0235,0,10000293
-0,2013-05-18,FP-non GTC,88750,0,-99,84.8856,0,10000147
-0,2013-01-15,ABIN,95672,0,15,86.0386,0,10000590
-0,2012-12-03,ABIN,50508,0,11,67.9981,0,10000582
-0,2012-06-19,FP-GTC,148324,15,15,53.4811,0,10000793
-0,2013-10-06,Auction,6762,0,14,73.5338,0,10000544
-0,2013-12-18,ABIN,63861,0,5,50.2892,0,10000769
-0,2013-12-04,Auction,25147,0,5,48.3303,0,10000590
-0,2012-04-25,FP-non GTC,62179,0,14,72.3156,0,10000102
-0,2013-11-20,Auction,24760,0,14,91.4883,0,10000370
-0,2013-02-08,Others,20485,0,11,29.1083,0,10000261
-0,2013-04-18,Others,2023,0,5,48.4321,0,10000355
-0,2012-11-21,Others,32876,0,12,16.7365,0,10000352
-0,2012-03-07,FP-GTC,61323,0,-99,32.5836,0,10000494
-0,2012-01-20,FP-non GTC,175750,0,16,52.8153,0,10000025
-0,2013-10-06,FP-GTC,26249,0,-99,1.9539,0,10000660
-0,2012-01-10,Auction,57990,0,14,57.5800,0,10000830
-0,2013-09-13,Others,46575,0,15,60.3315,0,10000528
-0,2012-11-19,FP-non GTC,44079,0,11,73.0457,0,10000482
-0,2012-09-23,Auction,95672,0,13,84.6345,0,10000586
-0,2013-08-31,Others,216,0,12,34.7429,0,10000676
-0,2013-04-11,FP-GTC,963,0,-99,77.7226,0,10000367
-0,2013-03-28,ABIN,15687,0,-99,8.2384,0,10000542
-0,2012-05-24,ABIN,152801,0,15,39.0204,0,10000277
-0,2013-03-05,ABIN,67698,0,16,40.6091,0,10000886
-0,2013-08-06,Auction,13987,0,13,36.5088,0,10000628
-0,2013-02-04,ABIN,20213,0,11,73.5301,0,10000134
-0,2012-05-19,FP-non GTC,38238,0,12,59.3237,0,10000500
-0,2013-08-19,ABIN,57990,0,12,10.1688,0,10000715
-0,2012-03-28,FP-non GTC,46575,0,16,77.7396,0,10000756
-0,2012-02-19,Auction,43479,0,-99,45.2450,0,10000049
-0,2012-04-02,FP-GTC,41940,0,15,58.8415,0,10000195
-0,2013-08-24,Others,16145,0,11,5.4971,0,10000119
-0,2013-01-10,Auction,106340,15,12,24.1336,0,10000603
-0,2012-09-23,Auction,152801,0,5,70.4024,0,10000384
-0,2013-07-29,FP-GTC,73506,0,16,14.5932,0,10000580
-0,2012-02-16,FP-non GTC,67698,0,5,4.4981,0,10000524
-0,2013-01-06,Others,65,0,13,64.8343,0,10000731
-0,2013-03-02,Auction,43398,0,16,22.6621,0,10000734
-0,2013-10-25,Auction,57990,0,13,64.3384,0,10000961
-0,2012-07-07,FP-GTC,156356,0,16,47.3744,0,10000795
-0,2013-12-19,FP-GTC,67698,0,16,96.9535,0,10000207
-0,2013-11-04,Auction,63861,0,5,34.8173,0,10000964
-0,2013-01-18,FP-non GTC,174053,3,5,94.6036,0,10000393
-0,2013-01-30,Auction,43479,0,16,28.9982,0,10000854
-0,2012-05-16,ABIN,24541,0,15,91.6347,0,10000536
-0,2013-06-03,FP-non GTC,159184,0,15,27.9722,0,10000644
-0,2013-08-24,FP-GTC,2635,0,14,95.7894,0,10000503
-0,2012-07-02,FP-non GTC,16509,0,5,76.9646,0,10000320
-0,2013-09-09,ABIN,80287,0,16,68.9620,0,10000934
-0,2012-04-15,FP-GTC,156356,0,12,95.3640,0,10000269
-0,2013-07-10,FP-GTC,1349,0,13,82.7969,0,10000033
-0,2012-12-02,FP-GTC,963,0,13,31.4196,0,10000717
-0,2013-02-17,Auction,26249,0,14,9.5202,0,10000543
-0,2013-02-17,Others,87118,0,16,77.3899,0,10000647
-0,2012-12-12,Others,6762,0,15,76.3422,0,10000076
-0,2013-07-08,Others,80053,0,5,95.2412,0,10000723
-0,2012-08-19,ABIN,32876,0,12,40.9587,0,10000352
-0,2013-07-24,Auction,314,0,11,78.8147,0,10000996
-0,2013-10-10,Others,57990,0,12,89.8124,0,10000549
-0,2013-11-13,Others,139973,0,16,85.5451,0,10000089
-0,2013-03-29,FP-non GTC,25147,0,16,16.2290,0,10000017
-0,2012-06-04,FP-non GTC,44079,0,-99,0.1588,0,10000758
-0,2012-07-19,FP-non GTC,57784,0,12,84.9441,0,10000553
-0,2012-01-08,FP-GTC,155226,0,15,31.9705,0,10000778
-0,2012-09-13,FP-non GTC,963,0,16,39.6392,0,10000799
-0,2012-12-24,Others,175750,0,15,77.3617,0,10000164
-0,2013-08-31,Auction,57990,0,12,76.7885,0,10000580
-0,2013-08-06,FP-non GTC,156356,0,13,86.0218,0,10000445
-0,2013-05-29,FP-non GTC,11848,0,11,23.0269,0,10000372
-0,2013-05-19,Others,53064,0,13,66.3990,0,10000390
-0,2012-11-01,FP-non GTC,51582,0,14,57.6157,0,10000061
-0,2013-01-01,Auction,1357,0,-99,18.2117,0,10000860
-0,2013-05-17,Others,121153,0,14,79.4534,0,10000155
-0,2013-07-04,FP-GTC,80053,0,5,43.3271,0,10000456
-0,2013-12-17,Others,165888,0,-99,9.8402,0,10000357
-0,2013-11-28,FP-GTC,26262,0,-99,1.4013,0,10000731
-0,2013-02-08,FP-GTC,65,0,11,61.9604,0,10000176
-0,2012-05-17,ABIN,9426,3,12,68.0170,0,10000895
-0,2012-08-29,Auction,159184,0,13,43.7406,0,10000723
-0,2013-10-06,FP-GTC,61323,0,14,85.9325,0,10000692
-0,2013-08-08,FP-GTC,20886,0,14,88.0137,0,10000063
-0,2013-04-06,FP-non GTC,156356,0,16,67.1201,0,10000441
-0,2012-03-09,Auction,1504,0,13,45.8878,0,10000862
-0,2013-12-13,FP-GTC,139973,3,16,58.7879,0,10000947
-0,2013-02-03,ABIN,95173,0,-99,33.2714,0,10000634
-0,2013-02-19,Others,41940,0,5,32.2179,0,10000257
-0,2012-05-09,ABIN,65,0,11,96.0579,0,10000675
-0,2012-10-30,Auction,80135,0,11,55.5335,0,10000288
-0,2013-07-14,ABIN,11554,0,16,9.7012,0,10000210
-0,2012-07-29,Others,11554,0,5,91.5938,0,10000736
-0,2012-06-03,ABIN,80287,0,13,99.7431,0,10000403
-0,2012-10-10,Auction,36250,0,13,14.7056,0,10000561
-0,2013-06-20,ABIN,20485,0,5,23.9120,0,10000798
-0,2013-12-26,Others,11554,0,15,27.2028,0,10000154
-0,2013-05-07,Auction,11848,0,-99,73.7354,0,10000448
-0,2013-12-14,ABIN,61323,0,16,44.6086,0,10000555
-0,2012-08-29,FP-GTC,139973,0,14,34.7404,0,10000686
-0,2013-12-31,FP-GTC,100847,0,15,70.6027,0,10000534
-0,2013-12-13,FP-non GTC,16145,3,12,76.5802,0,10000864
-0,2012-01-24,FP-non GTC,57990,0,-99,54.7904,0,10000325
-0,2013-07-26,Others,20485,0,11,9.3130,0,10000991
-0,2013-09-30,ABIN,31519,0,15,95.7797,0,10000701
-0,2012-03-30,FP-non GTC,88750,0,16,34.6777,0,10000494
-0,2012-05-11,Others,20485,0,16,2.2782,0,10000436
-0,2012-11-21,FP-non GTC,156356,0,13,3.4276,0,10000574
-0,2012-11-19,FP-non GTC,36250,0,16,42.1665,0,10000885
-0,2013-01-07,Others,20485,0,13,29.1424,0,10000931
-0,2013-01-16,ABIN,88750,0,13,87.3858,0,10000276
-0,2012-07-20,ABIN,25147,0,-99,41.9809,0,10000928
-0,2013-06-27,FP-non GTC,75665,0,11,65.3889,0,10000587
-0,2013-12-04,FP-non GTC,45333,0,15,41.7577,0,10000979
-0,2012-05-01,Auction,4943,0,11,76.3978,0,10000385
-0,2013-05-03,Others,66767,15,-99,69.0620,0,10000186
-0,2013-02-14,FP-non GTC,95672,0,14,15.6152,0,10000356
-0,2012-01-01,Others,1357,0,13,16.7271,0,10000263
-0,2012-01-05,FP-non GTC,67703,3,5,32.2005,0,10000141
-0,2013-12-15,Others,2023,0,-99,55.3349,0,10000920
-0,2012-10-02,FP-GTC,314,0,14,95.9217,0,10000647
-0,2012-10-27,Others,64076,0,5,3.4266,0,10000486
-0,2013-10-17,Others,41940,0,15,3.0363,0,10000299
-0,2013-02-09,Others,13836,0,15,37.1746,0,10000799
-0,2013-04-03,FP-GTC,31519,0,15,83.3031,0,10000656
-0,2013-08-30,Others,63861,0,13,37.8620,0,10000958
-0,2012-01-01,Others,1349,0,13,0.7227,0,10000774
-0,2013-03-24,FP-non GTC,87118,0,13,32.5615,0,10000661
-0,2012-12-03,FP-non GTC,216,0,5,40.1447,0,10000923
-0,2012-07-12,FP-GTC,94847,0,5,74.4586,0,10000161
-0,2013-03-18,Others,1161,3,-99,70.1521,0,10000123
-0,2013-11-17,Auction,165888,0,5,16.1344,0,10000029
-0,2013-11-25,FP-non GTC,44079,0,5,18.0996,0,10000461
-0,2012-12-06,FP-GTC,159184,0,15,63.6868,0,10000399
-0,2013-09-27,FP-non GTC,161567,15,15,35.8966,0,10000283
-0,2013-11-02,ABIN,20865,0,5,70.2863,0,10000724
-0,2013-04-29,Auction,314,0,13,53.6862,0,10000676
-0,2013-01-21,Auction,20886,0,-99,32.5548,0,10000052
-0,2012-04-18,FP-GTC,67698,0,14,90.8206,0,10000221
-0,2013-10-14,Others,46575,0,13,92.8392,0,10000187
-0,2013-07-06,FP-GTC,26249,0,-99,54.6729,0,10000651
-0,2012-09-15,Others,16145,0,15,70.0461,0,10000279
-0,2013-11-02,Others,38238,0,11,22.9595,0,10000451
-0,2012-06-03,ABIN,57013,0,11,28.9005,0,10000028
-0,2013-10-08,FP-GTC,20886,0,14,24.6958,0,10000676
-0,2013-04-11,Auction,100847,0,14,42.1878,0,10000645
-0,2013-11-07,Auction,45333,0,13,58.1089,0,10000182
-0,2012-10-18,FP-non GTC,57990,0,15,37.9049,0,10000696
-0,2012-11-28,FP-non GTC,63889,0,5,59.1268,0,10000116
-0,2012-03-02,Auction,45333,0,13,37.4799,0,10000298
-0,2013-07-27,FP-non GTC,75665,0,12,19.4230,0,10000333
-0,2012-12-01,Auction,44079,0,11,95.4596,0,10000163
-0,2012-04-11,FP-non GTC,161567,15,13,47.4582,0,10000082
-0,2013-06-23,ABIN,175750,3,13,23.9511,0,10000560
-0,2012-02-17,Others,46575,0,15,25.7613,0,10000308
-0,2013-05-19,ABIN,223,0,15,64.7127,0,10000850
-0,2012-10-29,FP-non GTC,159184,0,5,15.4188,0,10000995
-0,2012-07-17,Auction,4943,0,11,32.1734,0,10000820
-0,2013-01-03,ABIN,57784,0,5,47.7542,0,10000563
-0,2013-07-17,FP-GTC,150047,3,16,35.0905,0,10000545
-0,2013-07-20,Others,87118,0,14,24.6574,0,10000985
-0,2013-12-14,Others,11848,0,14,80.9504,0,10000318
-0,2013-01-16,Auction,106246,0,12,94.2201,0,10000113
-0,2012-11-04,Others,63861,0,12,71.8354,0,10000985
-0,2012-08-25,FP-non GTC,100847,0,5,68.7418,0,10000581
-0,2013-07-22,Auction,156356,0,15,63.6314,0,10000140
-0,2013-11-04,Auction,67698,0,16,15.6061,0,10000128
-0,2013-03-01,FP-GTC,20485,0,-99,49.2253,0,10000722
-0,2013-12-03,FP-GTC,46575,0,-99,71.5261,0,10000796
-0,2013-03-25,FP-non GTC,60340,0,11,23.0796,0,10000605
-0,2012-11-16,FP-non GTC,99985,0,11,91.6223,0,10000619
-0,2012-05-24,Auction,16509,0,16,97.7044,0,10000002
-0,2012-04-04,ABIN,139973,3,13,16.7393,0,10000923
-0,2012-07-20,FP-GTC,174106,3,13,8.8874,0,10000134
-0,2013-09-12,Auction,33038,15,13,46.7475,0,10000154
-0,2012-06-12,ABIN,57990,0,11,33.1479,0,10000036
-0,2013-08-31,ABIN,46575,0,12,41.0635,0,10000310
-0,2012-12-18,Auction,1349,0,5,14.6328,0,10000296
-0,2012-02-18,ABIN,16509,0,-99,76.3163,0,10000486
-0,2013-07-02,Auction,3838,0,11,95.7402,0,10000963
-0,2012-06-28,Auction,145970,0,16,41.1425,0,10000646
-0,2013-11-03,FP-GTC,12688,0,5,73.9106,0,10000357
-0,2012-03-30,FP-non GTC,16509,0,5,39.5703,0,10000495
-0,2012-08-18,FP-non GTC,75665,0,11,61.4928,0,10000522
-0,2013-07-29,Others,156356,0,16,98.2405,0,10000417
-0,2013-07-19,Others,32876,0,12,68.5148,0,10000306
-0,2012-02-13,ABIN,175750,0,16,64.5444,0,10000274
-0,2013-02-04,FP-non GTC,13836,0,-99,65.7663,0,10000511
-0,2013-07-24,FP-GTC,165888,0,16,91.7808,0,10000330
-0,2012-04-25,Others,41940,0,-99,6.5053,0,10000357
-0,2013-11-28,ABIN,106340,15,12,71.4264,0,10000155
-0,2012-09-16,FP-non GTC,170083,3,-99,80.2442,0,10000690
-0,2012-05-10,FP-GTC,16145,0,14,11.0775,0,10000372
-0,2013-10-28,FP-non GTC,15687,0,16,65.9436,0,10000032
-0,2013-05-17,FP-non GTC,94847,0,13,37.3520,0,10000044
-0,2013-05-31,FP-GTC,223,0,5,93.3181,0,10000365
-0,2012-01-07,FP-non GTC,73506,0,13,13.1225,0,10000218
-0,2013-07-01,FP-GTC,64076,0,15,78.1094,0,10000081
-0,2013-07-26,Others,67698,0,11,14.9245,0,10000775
-0,2013-03-02,Auction,75665,0,5,84.2445,0,10000162
-0,2012-01-13,FP-GTC,1120,3,16,43.8855,0,10000157
-0,2012-06-02,ABIN,53064,0,12,17.3600,0,10000091
-0,2012-02-23,ABIN,31387,3,14,43.1757,0,10000071
-0,2013-10-22,FP-non GTC,158798,0,15,10.9759,0,10000341
-0,2013-05-23,FP-GTC,11848,0,-99,7.5230,0,10000277
-0,2013-07-25,FP-GTC,43479,0,14,13.2620,0,10000500
-0,2012-03-03,FP-non GTC,11848,0,14,95.0683,0,10000164
-0,2012-01-09,ABIN,11554,0,15,6.8777,0,10000785
-0,2013-05-08,ABIN,80287,0,14,95.5320,0,10000595
-0,2013-07-26,FP-GTC,121153,0,12,28.7213,0,10000829
-0,2012-11-27,FP-GTC,31673,0,14,7.8033,0,10000498
-0,2013-06-04,ABIN,43398,0,15,76.0187,0,10000061
-0,2012-10-05,ABIN,15687,0,13,87.1235,0,10000649
-0,2013-06-29,Auction,61323,0,14,78.4000,0,10000765
-0,2012-07-25,ABIN,216,0,12,43.1556,0,10000597
-0,2013-01-03,FP-non GTC,121153,0,12,70.1009,0,10000201
-0,2013-10-22,FP-non GTC,170302,15,12,92.5959,0,10000151
-0,2012-02-03,FP-non GTC,155226,0,12,68.6486,0,10000870
-0,2013-03-30,FP-GTC,139973,0,11,93.3324,0,10000585
-0,2012-11-21,Others,20865,0,11,68.0525,0,10000115
-0,2012-11-09,ABIN,11848,0,13,66.3745,0,10000652
-0,2012-10-22,Auction,2023,0,13,52.4918,0,10000836
-0,2013-01-07,Auction,164261,0,-99,56.4680,0,10000560
-0,2012-01-05,Others,80053,0,-99,34.1305,0,10000905
-0,2012-08-27,FP-GTC,64076,0,14,3.0691,0,10000389
-0,2013-06-23,Others,139973,0,16,89.7673,0,10000636
-0,2013-09-13,Auction,67698,0,15,62.9192,0,10000596
-0,2012-08-18,FP-GTC,43398,0,12,45.3384,0,10000382
-0,2012-10-27,FP-non GTC,1120,3,5,84.6466,0,10000487
-0,2012-05-24,Auction,24541,0,14,56.7041,0,10000366
-0,2013-02-12,Auction,64076,0,13,1.1360,0,10000089
-0,2012-02-19,FP-non GTC,43398,0,14,7.2416,0,10000129
-0,2012-07-30,ABIN,48027,0,11,41.6704,0,10000761
-0,2013-10-04,FP-non GTC,37831,0,16,31.9567,0,10000610
-0,2013-11-18,Auction,223,0,16,21.8155,0,10000079
-0,2012-06-08,Others,80135,0,16,90.0166,0,10000478
-0,2012-10-25,Others,159184,0,16,92.0436,0,10000166
-0,2013-10-22,ABIN,13987,0,15,62.5546,0,10000592
-0,2013-11-01,ABIN,103324,15,16,14.9475,0,10000924
-0,2013-03-21,FP-GTC,1120,3,-99,29.5252,0,10000758
-0,2012-03-13,Others,759,0,16,62.4158,0,10000810
-0,2012-05-14,Auction,95672,0,16,92.0781,0,10000609
-0,2013-09-28,FP-GTC,64076,0,12,37.3599,0,10000487
-0,2013-10-09,ABIN,60606,3,13,48.9467,0,10000007
-0,2013-04-21,ABIN,26249,0,14,67.1520,0,10000257
-0,2013-05-08,FP-GTC,20886,0,13,19.5685,0,10000122
-0,2013-12-29,ABIN,1357,0,12,89.9069,0,10000900
-0,2013-02-14,FP-GTC,63889,0,16,74.5657,0,10000676
-0,2012-02-25,Auction,88750,0,-99,92.3939,0,10000769
-0,2012-06-22,Others,13836,0,14,77.1277,0,10000795
-0,2012-01-30,FP-GTC,43479,0,15,48.2346,0,10000013
-0,2013-08-17,ABIN,57990,3,-99,87.0638,0,10000277
-0,2013-06-24,FP-GTC,1357,0,15,75.0985,0,10000056
-0,2013-04-08,ABIN,57013,0,11,21.4485,0,10000195
-0,2012-04-07,FP-non GTC,24541,0,-99,40.6197,0,10000313
-0,2012-07-26,Others,963,0,16,99.8137,0,10000257
-0,2012-07-18,Others,15115,0,-99,9.9153,0,10000937
-0,2012-01-04,Auction,63889,0,13,84.5004,0,10000467
-0,2013-05-25,Auction,1120,3,14,40.3618,0,10000076
-0,2013-03-30,FP-GTC,38238,0,12,94.9870,0,10000997
-0,2012-05-12,Auction,32876,0,-99,38.5800,0,10000943
-0,2012-11-21,FP-GTC,100847,0,14,7.7495,0,10000588
-0,2013-10-02,Auction,15868,0,5,92.2309,0,10000520
-0,2013-11-05,ABIN,31673,0,-99,75.9838,0,10000344
-0,2013-03-23,FP-non GTC,48904,0,-99,15.4260,0,10000090
-0,2012-12-27,FP-non GTC,43479,0,5,35.7689,0,10000066
-0,2013-01-11,FP-non GTC,223,0,12,5.9423,0,10000174
-0,2012-05-25,Auction,50508,0,16,81.5853,0,10000337
-0,2012-09-08,Others,6762,0,15,11.8473,0,10000718
-0,2013-07-08,Auction,139973,0,5,11.7025,0,10000537
-0,2013-04-23,ABIN,32876,0,12,40.4116,0,10000798
-0,2013-03-22,Auction,2023,0,15,89.8093,0,10000662
-0,2012-11-01,Others,75708,3,11,95.9527,0,10000776
-0,2012-01-29,ABIN,156356,0,16,64.6181,0,10000910
-0,2012-01-20,Others,2635,0,11,81.6196,0,10000893
-0,2012-05-27,FP-non GTC,94847,0,15,87.1713,0,10000121
-0,2013-12-17,Others,156614,0,14,87.7791,0,10000531
-0,2013-03-05,ABIN,80135,0,16,64.8966,0,10000452
-0,2013-01-26,ABIN,156356,0,16,59.0108,0,10000671
-0,2012-01-08,FP-non GTC,57990,0,15,56.3470,0,10000028
-0,2013-08-04,ABIN,13836,0,11,98.5679,0,10000481
-0,2013-04-04,Others,63861,0,13,14.3911,0,10000944
-0,2013-02-15,FP-GTC,2023,0,5,5.5878,0,10000816
-0,2013-05-05,FP-non GTC,175750,0,-99,80.9471,0,10000727
-0,2013-06-24,FP-GTC,38238,0,12,96.7825,0,10000202
-0,2013-11-04,ABIN,45333,0,14,57.9540,0,10000767
-0,2012-02-23,FP-non GTC,57990,0,5,16.4790,0,10000146
-0,2013-01-13,ABIN,158798,0,11,45.5371,0,10000133
-0,2013-11-16,ABIN,73506,0,16,35.7207,0,10000304
-0,2012-11-15,Auction,64076,0,15,66.0059,0,10000375
-0,2012-01-03,ABIN,156614,0,15,84.8759,0,10000095
-0,2012-09-22,Auction,57990,0,14,42.7313,0,10000953
-0,2013-11-03,FP-non GTC,67698,0,11,8.9322,0,10000913
-0,2012-08-31,ABIN,1504,0,5,34.3706,0,10000748
-0,2013-12-13,FP-GTC,15868,0,-99,18.3204,0,10000495
-0,2012-07-20,ABIN,533,0,11,24.9451,0,10000237
-0,2012-07-14,ABIN,41940,0,11,42.9796,0,10000328
-0,2013-11-06,Others,32876,0,15,4.2693,0,10000383
-0,2012-07-02,Auction,41940,0,12,19.2987,0,10000196
-0,2012-06-21,FP-GTC,20865,0,-99,36.5029,0,10000098
-0,2012-09-19,ABIN,159184,0,15,65.9768,0,10000517
-0,2012-02-11,Others,16145,0,14,41.4762,0,10000733
-0,2012-07-19,Auction,63861,0,11,30.7294,0,10000351
-0,2013-05-24,ABIN,1349,0,12,61.0346,0,10000078
-0,2013-06-26,Auction,139973,3,11,49.3849,0,10000537
-0,2013-11-30,Others,1357,0,16,62.0388,0,10000532
-0,2012-11-01,Others,121153,0,5,64.8814,0,10000820
-0,2013-10-20,Auction,20485,0,5,88.9093,0,10000467
-0,2013-04-15,FP-GTC,43479,0,13,72.5969,0,10000352
-0,2012-04-02,FP-GTC,33977,15,16,42.1845,0,10000614
-0,2012-03-20,ABIN,2023,0,16,95.0254,0,10000014
-0,2013-02-21,FP-non GTC,57990,0,5,25.0404,0,10000386
-0,2013-06-02,FP-non GTC,6762,0,12,60.3684,0,10000783
-0,2012-01-07,FP-GTC,759,0,11,80.1720,0,10000952
-0,2012-12-01,FP-non GTC,533,0,12,47.1653,0,10000550
-0,2012-11-03,ABIN,99985,0,16,40.8280,0,10000629
-0,2013-10-15,FP-GTC,64076,0,15,55.9541,0,10000531
-0,2013-12-30,Auction,80135,0,13,13.2503,0,10000425
-0,2012-06-28,FP-GTC,15868,0,12,76.8360,0,10000557
-0,2013-04-07,FP-GTC,10866,0,16,37.7540,0,10000810
-0,2013-08-03,FP-GTC,139255,15,13,70.8704,0,10000514
-0,2012-07-23,ABIN,121153,0,5,65.6362,0,10000166
-0,2013-05-04,FP-GTC,31519,0,5,38.9751,0,10000932
-0,2013-01-27,ABIN,175750,3,-99,81.7736,0,10000529
-0,2013-03-30,ABIN,11554,0,15,37.8703,0,10000509
-0,2012-06-03,FP-GTC,20485,0,-99,62.4729,0,10000404
-0,2012-05-09,FP-non GTC,63861,0,11,35.1334,0,10000175
-0,2013-12-25,FP-non GTC,6762,0,12,95.0363,0,10000101
-0,2012-11-18,FP-GTC,175750,0,5,95.8938,0,10000537
-0,2013-01-07,FP-non GTC,57784,0,14,0.2995,0,10000847
-0,2012-03-11,Auction,63861,0,12,58.1028,0,10000912
-0,2012-03-23,Others,80287,0,14,24.7900,0,10000967
-0,2012-03-19,ABIN,99985,0,16,5.9564,0,10000684
-0,2013-11-24,Auction,20886,0,15,28.1384,0,10000239
-0,2012-04-04,FP-GTC,100847,0,11,91.4446,0,10000743
-0,2013-12-15,ABIN,44079,0,16,39.6860,0,10000476
-0,2013-10-06,FP-GTC,132939,0,12,20.8855,0,10000854
-0,2013-12-07,Others,25147,0,14,16.4357,0,10000198
-0,2012-02-04,Others,158798,0,13,44.5451,0,10000543
-0,2013-11-07,FP-GTC,10866,0,16,8.5249,0,10000105
-0,2012-08-04,FP-GTC,2635,0,12,11.7572,0,10000382
-0,2012-08-14,Auction,152801,0,15,46.6818,0,10000205
-0,2013-02-25,Auction,50508,0,15,37.8443,0,10000644
-0,2013-10-09,ABIN,223,0,13,58.8049,0,10000768
-0,2012-01-10,Auction,20485,0,11,92.2997,0,10000975
-0,2013-05-03,FP-non GTC,37831,0,12,15.9399,0,10000280
-0,2013-11-12,ABIN,73506,0,12,13.9272,0,10000332
-0,2013-08-16,Others,164,0,12,39.2371,0,10000887
-0,2013-10-27,FP-non GTC,15868,0,16,51.4769,0,10000303
-0,2013-04-27,FP-non GTC,80287,0,15,90.8971,0,10000207
-0,2013-11-06,FP-non GTC,11554,0,12,26.0123,0,10000855
-0,2013-11-25,Others,6762,0,14,96.0198,0,10000418
-0,2012-08-09,ABIN,80135,0,15,89.0544,0,10000861
-0,2013-09-28,ABIN,16509,0,12,90.2997,0,10000520
-0,2012-03-19,FP-GTC,43479,0,-99,93.8319,0,10000045
-0,2012-05-25,FP-GTC,139973,0,16,8.9816,0,10000684
-0,2013-04-04,Auction,25147,0,11,43.4557,0,10000801
-0,2013-07-07,Auction,67698,0,14,59.3634,0,10000305
-0,2013-10-27,ABIN,63861,0,14,48.1526,0,10000963
-0,2012-05-31,Auction,130,0,5,1.0560,0,10000357
-0,2012-08-21,Auction,13836,0,16,14.4247,0,10000278
-0,2013-07-24,Auction,15568,15,14,14.5748,0,10000590
-0,2013-04-05,ABIN,12688,0,12,2.0636,0,10000951
-0,2013-02-27,FP-non GTC,73506,0,14,2.7139,0,10000134
-0,2013-06-16,FP-non GTC,164,0,5,59.4198,0,10000745
-0,2012-04-01,Others,20213,0,11,77.8490,0,10000929
-0,2013-12-02,FP-non GTC,155226,0,16,71.1100,0,10000433
-0,2013-06-07,Auction,158798,0,15,98.6517,0,10000320
-0,2012-11-25,ABIN,31519,0,12,68.8974,0,10000726
-0,2013-04-17,FP-GTC,57013,0,14,49.4405,0,10000050
-0,2013-03-18,FP-non GTC,175750,0,13,2.3784,0,10000559
-0,2013-08-13,Others,62179,0,11,36.4940,0,10000652
-0,2012-10-15,ABIN,175750,0,14,42.7269,0,10000560
-0,2012-01-08,Others,6762,0,5,58.5212,0,10000829
-0,2013-05-20,FP-non GTC,216,0,14,16.9644,0,10000104
-0,2012-12-07,Others,51582,0,5,6.0617,0,10000228
-0,2013-12-04,ABIN,57013,0,16,48.2921,0,10000347
-0,2013-12-17,ABIN,759,0,14,12.8117,0,10000161
-0,2012-02-08,FP-non GTC,24760,0,13,11.3190,0,10000938
-0,2013-01-03,FP-non GTC,15568,15,14,0.3508,0,10000964
-0,2013-04-21,Others,87118,0,12,9.5385,0,10000949
-0,2013-11-13,Others,36250,0,12,0.9029,0,10000532
-0,2013-03-18,FP-GTC,164,0,12,3.7727,0,10000237
-0,2013-06-29,Others,15868,0,11,23.3421,0,10000671
-0,2013-12-25,Others,25147,0,5,99.4168,0,10000708
-0,2012-10-06,FP-GTC,533,0,12,40.5069,0,10000295
-0,2013-02-21,ABIN,216,0,14,84.3818,0,10000637
-0,2012-08-27,FP-non GTC,67698,0,16,7.0299,0,10000671
-0,2013-04-13,Others,51582,0,13,67.9730,0,10000778
-0,2013-07-10,FP-GTC,139973,0,13,24.8423,0,10000253
-0,2013-11-02,ABIN,1349,0,14,56.6310,0,10000922
-0,2013-05-11,Others,26249,0,16,49.7279,0,10000610
-0,2013-01-17,FP-non GTC,118687,3,11,34.6913,0,10000002
-0,2012-07-02,Others,148324,15,-99,90.6687,0,10000613
-0,2013-07-14,ABIN,41940,0,-99,60.2733,0,10000145
-0,2013-06-13,Auction,63889,0,11,87.9644,0,10000920
-0,2013-05-05,FP-non GTC,1120,3,16,99.4791,0,10000720
-0,2012-03-30,Others,4943,0,14,56.9643,0,10000105
-0,2012-01-13,FP-non GTC,139973,3,-99,15.8380,0,10000587
-0,2012-09-02,ABIN,12688,0,13,69.4513,0,10000756
-0,2013-08-08,Others,121153,0,11,29.2318,0,10000289
-0,2013-03-09,Others,99985,0,15,14.7661,0,10000453
-0,2012-10-17,FP-non GTC,25147,0,11,96.1909,0,10000054
-0,2012-01-20,Others,73506,0,14,56.0157,0,10000909
-0,2012-04-15,FP-GTC,16145,0,12,94.6354,0,10000551
-0,2013-08-08,Others,3838,0,13,85.7695,0,10000464
-0,2013-01-21,Others,139973,0,13,64.5021,0,10000107
-0,2012-06-17,Others,139973,0,-99,72.8065,0,10000495
-0,2013-12-31,Others,20886,0,15,91.2326,0,10000746
-0,2012-02-11,FP-non GTC,4943,0,14,28.3146,0,10000649
-0,2013-10-01,Auction,155226,0,15,72.2196,0,10000576
-0,2012-08-25,ABIN,223,0,15,17.6408,0,10000661
-0,2012-07-10,Others,67698,0,15,70.9451,0,10000115
-0,2013-12-27,FP-non GTC,26262,0,13,64.5694,0,10000740
-0,2012-02-05,Auction,32996,15,12,9.9521,0,10000833
-0,2012-11-07,FP-non GTC,25147,0,16,83.3886,0,10000494
-0,2014-01-01,Others,45333,0,16,19.2606,0,10000869
-0,2013-10-05,Auction,50508,0,14,39.1465,0,10000904
-0,2012-02-25,FP-GTC,37831,0,11,12.4616,0,10000680
-0,2012-05-16,Others,31519,0,12,66.9268,0,10000548
-0,2013-11-08,FP-GTC,100847,0,11,47.7510,0,10000699
-0,2012-03-15,ABIN,106246,0,16,96.1592,0,10000922
-0,2013-05-15,Auction,64076,0,15,37.7430,0,10000148
-0,2012-11-03,Auction,61323,0,15,14.3851,0,10000353
-0,2012-07-28,ABIN,175750,0,-99,45.5415,0,10000869
-0,2013-06-26,FP-non GTC,95672,0,14,90.7320,0,10000507
-0,2012-09-11,Others,223,0,15,10.5185,0,10000705
-0,2012-09-24,FP-GTC,43479,0,16,3.4611,0,10000991
-0,2013-09-13,FP-GTC,100847,0,-99,51.0213,0,10000871
-0,2013-07-01,Others,132939,0,16,14.6584,0,10000859
-0,2012-06-13,Others,26262,0,-99,46.0446,0,10000041
-0,2013-10-26,Auction,3838,0,12,24.8196,0,10000561
-0,2012-03-05,Others,26262,0,13,66.7748,0,10000859
-0,2012-12-10,FP-GTC,20485,0,12,52.3473,0,10000171
-0,2013-11-15,Others,2023,0,5,84.6955,0,10000451
-0,2013-12-12,ABIN,80135,0,11,95.0112,0,10000718
-0,2013-04-16,ABIN,106340,15,16,14.6367,0,10000822
-0,2012-03-10,Auction,161567,15,16,5.2800,0,10000081
-0,2013-09-19,Auction,11848,0,11,27.6166,0,10000388
-0,2013-12-28,FP-GTC,216,0,15,90.1257,0,10000122
-0,2012-01-26,ABIN,314,0,12,3.5634,0,10000912
-0,2013-12-13,FP-GTC,158798,0,5,41.6652,0,10000485
-0,2013-09-26,FP-non GTC,80135,0,11,20.5746,0,10000138
-0,2012-01-20,Auction,62179,0,14,94.2904,0,10000310
-0,2013-04-20,ABIN,963,0,5,68.3604,0,10000571
-0,2012-02-03,FP-GTC,164,0,11,94.0501,0,10000120
-0,2013-10-14,Others,2023,0,12,84.9792,0,10000371
-0,2013-08-19,FP-GTC,57990,3,14,57.0343,0,10000171
-0,2012-06-18,Others,963,0,12,44.4186,0,10000343
-0,2012-10-18,Others,31519,0,12,50.6753,0,10000875
-0,2013-07-09,Others,36250,0,16,99.5251,0,10000446
-0,2012-01-19,ABIN,38238,0,14,0.1259,0,10000242
-0,2013-05-19,FP-non GTC,13836,0,16,92.7668,0,10000895
-0,2013-01-29,Others,95672,0,5,13.6012,0,10000843
-0,2013-01-30,ABIN,67698,0,5,13.9968,0,10000424
-0,2012-09-02,FP-GTC,223,0,-99,38.0021,0,10000803
-0,2012-05-11,Auction,11848,0,13,7.1910,0,10000964
-0,2013-05-14,FP-non GTC,759,0,16,43.1720,0,10000123
-0,2013-03-06,ABIN,3838,0,16,59.8866,0,10000653
-0,2012-08-23,FP-GTC,20213,0,16,91.3442,0,10000615
-0,2013-03-24,FP-GTC,4943,0,-99,18.4365,0,10000718
-0,2012-01-26,FP-non GTC,11554,0,14,41.6729,0,10000174
-0,2013-10-07,Others,108782,15,14,14.9130,0,10000643
-0,2013-07-25,Others,13987,0,-99,28.7021,0,10000154
-0,2013-11-08,FP-non GTC,31673,0,13,83.8884,0,10000059
-0,2013-01-06,ABIN,82494,15,12,74.4582,0,10000023
-0,2013-01-21,FP-non GTC,13987,0,11,72.4276,0,10000541
-0,2013-11-22,Auction,67698,0,11,53.1908,0,10000553
-0,2012-02-02,FP-non GTC,175750,0,12,79.1160,0,10000121
-0,2012-10-14,FP-GTC,24760,0,11,70.5894,0,10000677
-0,2012-10-20,ABIN,73506,0,11,3.7214,0,10000076
-0,2013-12-01,Others,16145,0,15,54.8674,0,10000092
-0,2012-11-07,FP-non GTC,53064,0,14,53.4385,0,10000267
-0,2012-04-20,FP-non GTC,53064,0,15,76.5825,0,10000551
-0,2012-11-18,Auction,50677,0,11,44.5149,0,10000924
-0,2013-07-31,FP-GTC,51582,0,14,86.5325,0,10000652
-0,2012-12-25,Others,31387,3,13,79.7624,0,10000495
-0,2012-08-10,ABIN,2635,0,12,54.5975,0,10000453
-0,2012-06-09,Others,31519,0,15,20.1828,0,10000226
-0,2012-09-16,Others,44079,0,16,46.0626,0,10000193
-0,2012-04-23,Others,16145,0,11,60.2064,0,10000058
-0,2012-01-03,Others,164,0,15,94.1749,0,10000509
-0,2012-05-16,FP-GTC,13987,0,15,5.6834,0,10000052
-0,2012-07-24,FP-non GTC,16145,0,15,8.1625,0,10000036
-0,2013-01-01,FP-non GTC,32996,15,15,75.5057,0,10000393
-0,2013-01-28,FP-non GTC,155226,0,16,49.2395,0,10000966
-0,2013-03-19,FP-GTC,20485,0,5,57.4433,0,10000138
-0,2013-02-03,FP-GTC,57784,0,5,11.8152,0,10000731
-0,2012-01-02,Others,53064,0,-99,45.8162,0,10000785
-0,2013-02-23,ABIN,16145,0,14,25.1092,0,10000582
-0,2013-04-19,Auction,159184,0,12,85.3510,0,10000422
-0,2012-10-12,FP-non GTC,139973,0,11,95.7331,0,10000791
-0,2012-04-20,FP-non GTC,87118,0,16,15.2148,0,10000621
-0,2013-04-30,Auction,53064,0,5,89.5035,0,10000377
-0,2013-10-24,FP-non GTC,73506,0,12,84.0199,0,10000503
-0,2012-01-08,Auction,63861,0,13,74.8860,0,10000589
-0,2012-05-21,Others,63861,3,16,63.7247,0,10000730
-0,2013-01-29,Auction,24541,0,15,22.2189,0,10000167
-0,2012-02-08,Auction,533,0,5,53.5867,0,10000564
-0,2012-08-11,FP-non GTC,174053,3,13,0.5792,0,10000568
-0,2012-07-05,FP-GTC,20886,0,12,31.1464,0,10000806
-0,2012-01-08,Auction,156356,0,5,7.4293,0,10000638
-0,2012-02-14,Others,65,0,14,62.6745,0,10000253
-0,2013-03-25,FP-non GTC,48027,0,-99,53.4890,0,10000899
-0,2012-09-30,Others,57013,0,12,75.9109,0,10000274
-0,2013-09-01,ABIN,32876,0,12,75.0498,0,10000086
-0,2013-11-11,FP-GTC,164,0,12,36.6051,0,10000521
-0,2012-02-18,FP-GTC,57013,0,12,14.3117,0,10000763
-0,2013-11-23,ABIN,139973,0,15,58.3344,0,10000787
-0,2012-03-21,FP-non GTC,41940,0,15,38.2502,0,10000342
-0,2012-02-24,Auction,57990,0,5,73.5795,0,10000175
-0,2013-07-20,Auction,88750,0,-99,58.2532,0,10000122
-0,2012-10-17,Others,43479,0,13,67.9585,0,10000516
-0,2012-11-03,FP-non GTC,118687,3,13,90.9173,0,10000957
-0,2012-10-22,FP-GTC,60340,0,14,68.5877,0,10000502
-0,2013-04-25,FP-GTC,11554,0,5,60.8280,0,10000074
-0,2012-10-23,FP-GTC,24760,0,13,6.0732,0,10000138
-0,2013-04-29,ABIN,175750,0,11,8.0341,0,10000077
-0,2012-05-06,Auction,106340,15,14,8.0384,0,10000234
-0,2012-07-15,ABIN,57784,0,14,27.1882,0,10000729
-0,2012-10-15,Auction,67698,0,-99,63.1790,0,10000226
-0,2013-03-22,FP-non GTC,57013,0,14,27.7251,0,10000658
-0,2013-08-12,FP-non GTC,11848,0,13,38.5597,0,10000020
-0,2012-07-14,FP-GTC,80053,0,-99,81.0002,0,10000850
-0,2012-08-12,FP-non GTC,38238,0,11,58.6987,0,10000377
-0,2013-09-09,FP-GTC,106246,0,11,5.0352,0,10000594
-0,2013-02-11,FP-non GTC,57784,0,12,80.8420,0,10000657
-0,2013-03-09,Others,175750,0,14,87.9884,0,10000152
-0,2013-12-18,FP-non GTC,20485,0,14,74.4627,0,10000272
-0,2013-03-16,FP-non GTC,106246,0,5,64.1494,0,10000459
-0,2012-09-06,FP-non GTC,80135,0,13,30.1506,0,10000269
-0,2013-06-18,FP-non GTC,80135,0,-99,75.5826,0,10000706
-0,2013-03-16,FP-GTC,156614,0,15,36.7157,0,10000761
-0,2012-11-07,Others,1349,0,15,96.7575,0,10000612
-0,2012-11-25,ABIN,95173,0,5,61.1186,0,10000364
-0,2013-08-10,ABIN,31673,0,11,63.4095,0,10000421
-0,2012-02-17,ABIN,88750,0,14,87.8624,0,10000306
-0,2013-03-29,ABIN,139973,0,16,9.1685,0,10000089
-0,2012-12-27,ABIN,121153,0,12,88.7482,0,10000726
-0,2013-10-13,FP-GTC,67698,0,16,20.8225,0,10000504
-0,2012-09-26,ABIN,2023,0,11,83.6499,0,10000641
-0,2012-08-01,Others,139973,0,-99,1.0542,0,10000202
-0,2012-03-02,ABIN,113802,15,5,23.4439,0,10000068
-0,2012-10-25,Others,10866,0,11,60.7453,0,10000770
-0,2013-01-07,FP-non GTC,20485,0,16,16.4177,0,10000698
-0,2012-03-18,FP-non GTC,46575,0,15,55.2768,0,10000962
-0,2012-07-09,FP-GTC,106246,0,14,99.6346,0,10000563
-0,2012-09-26,FP-GTC,314,0,13,45.8302,0,10000769
-0,2012-01-14,FP-non GTC,15868,0,16,47.5496,0,10000869
-0,2012-01-01,ABIN,63861,0,13,76.1393,0,10000028
-0,2012-08-15,FP-GTC,26249,0,5,40.7448,0,10000419
-0,2012-09-24,FP-non GTC,43398,0,12,54.0864,0,10000688
-0,2013-10-13,Others,533,0,-99,58.0312,0,10000658
-0,2012-07-13,FP-GTC,148324,15,11,71.6067,0,10000006
-0,2012-01-21,FP-GTC,32996,15,11,86.8077,0,10000928
-0,2012-04-04,Others,63861,3,16,85.2671,0,10000648
-0,2012-10-20,FP-non GTC,31519,0,13,99.1930,0,10000469
-0,2013-12-28,FP-GTC,1504,0,5,14.9821,0,10000342
-0,2013-06-19,Others,25147,0,5,77.4671,0,10000212
-0,2013-05-04,Others,13836,0,11,13.4655,0,10000331
-0,2012-06-28,ABIN,145970,0,15,74.3304,0,10000900
-0,2012-12-24,FP-non GTC,156614,0,12,79.4508,0,10000743
-0,2013-07-17,FP-GTC,139973,0,13,30.0555,0,10000949
-0,2013-11-23,Others,31519,3,11,9.0845,0,10000883
-0,2013-01-15,Others,164261,0,12,31.5563,0,10000029
-0,2012-04-15,FP-non GTC,20213,0,-99,55.6917,0,10000513
-0,2013-01-20,FP-non GTC,24760,0,15,71.7279,0,10000274
-0,2012-11-13,FP-GTC,95173,0,5,95.7043,0,10000418
-0,2012-09-03,ABIN,57990,0,12,1.5845,0,10000703
-0,2012-01-02,FP-non GTC,57990,3,16,12.3706,0,10000090
-0,2013-12-06,Auction,67703,3,5,70.4134,0,10000753
-0,2012-06-09,Others,50677,0,12,88.7297,0,10000990
-0,2012-05-17,ABIN,31519,0,14,37.7070,0,10000326
-0,2012-03-03,Others,156356,0,16,50.7844,0,10000693
-0,2012-05-21,Auction,44079,0,5,86.7590,0,10000795
-0,2013-10-15,Auction,26262,0,15,56.3327,0,10000077
-0,2013-12-27,ABIN,20213,0,13,96.1815,0,10000195
-0,2012-02-15,ABIN,314,0,5,6.5869,0,10000753
-0,2012-08-17,FP-non GTC,6762,0,15,41.0782,0,10000797
-0,2013-04-29,Auction,145970,0,15,11.0876,0,10000208
-0,2013-10-27,Auction,80053,0,14,81.7435,0,10000310
-0,2012-12-21,FP-GTC,63861,0,13,57.9491,0,10000971
-0,2013-07-29,ABIN,45333,0,12,33.5405,0,10000475
-0,2012-04-18,Auction,64076,0,13,83.3621,0,10000697
-0,2012-12-14,FP-non GTC,63889,0,16,0.9640,0,10000607
-0,2013-06-07,Auction,145970,0,13,6.1316,0,10000450
-0,2012-11-27,Others,164,0,15,53.7718,0,10000305
-0,2012-08-03,ABIN,61323,0,15,62.2945,0,10000214
-0,2012-07-07,ABIN,75665,0,15,77.0551,0,10000094
-0,2012-08-06,FP-non GTC,99985,0,13,39.1253,0,10000609
-0,2012-10-01,Auction,759,0,15,69.1790,0,10000199
-0,2012-03-22,Auction,20865,0,5,40.7800,0,10000392
-0,2012-02-06,FP-non GTC,132939,0,11,66.8705,0,10000207
-0,2012-10-17,FP-non GTC,63861,3,14,35.8018,0,10000096
-0,2013-02-21,Others,41940,0,5,87.4721,0,10000858
-0,2012-11-16,FP-GTC,57784,0,15,96.2799,0,10000537
-0,2012-01-10,Others,158666,15,11,7.4077,0,10000190
-0,2013-10-29,FP-non GTC,65,0,-99,21.3439,0,10000182
-0,2013-07-30,Auction,103178,15,-99,94.8679,0,10000943
-0,2012-08-05,FP-non GTC,175750,0,14,10.0980,0,10000212
-0,2012-07-29,ABIN,24760,0,13,44.5270,0,10000168
-0,2013-04-27,FP-GTC,57990,3,-99,75.6664,0,10000878
-0,2012-12-06,FP-GTC,37831,0,11,57.1191,0,10000090
-0,2013-08-29,FP-GTC,164,0,16,75.2345,0,10000689
-0,2012-08-30,FP-GTC,63861,0,12,21.2099,0,10000643
-0,2013-05-30,ABIN,57013,0,15,28.8410,0,10000290
-0,2013-10-17,ABIN,50677,0,-99,16.3836,0,10000705
-0,2012-04-25,FP-GTC,67698,0,11,52.3502,0,10000257
-0,2013-08-18,ABIN,759,0,16,23.8258,0,10000911
-0,2013-02-24,Others,13836,0,-99,40.6826,0,10000399
-0,2012-07-22,FP-non GTC,158798,0,15,94.1886,0,10000705
-0,2012-08-05,FP-GTC,100847,0,-99,74.1500,0,10000797
-0,2012-12-30,Auction,31519,0,11,69.1499,0,10000576
-0,2013-05-18,Auction,63864,3,14,44.5408,0,10000232
-0,2012-11-01,ABIN,53064,0,15,19.9817,0,10000065
-0,2013-06-25,FP-GTC,156614,0,16,91.4652,0,10000326
-0,2013-07-11,FP-non GTC,10058,3,5,92.6671,0,10000787
-0,2013-01-19,FP-non GTC,13987,0,12,49.0313,0,10000556
-0,2013-06-17,Auction,94847,0,13,95.5377,0,10000047
-0,2012-12-23,Others,64076,0,-99,50.1173,0,10000063
-0,2013-01-07,FP-non GTC,67698,0,5,6.5979,0,10000412
-0,2012-01-20,ABIN,175750,3,16,79.6931,0,10000270
-0,2013-01-14,Others,95672,3,12,31.7315,0,10000044
-0,2013-02-22,Others,26249,0,16,43.7969,0,10000634
-0,2013-04-13,FP-non GTC,156614,0,16,79.3213,0,10000321
-0,2013-09-18,FP-non GTC,43398,0,5,17.3702,0,10000621
-0,2013-09-05,ABIN,67703,3,11,55.6439,0,10000784
-0,2013-05-25,FP-GTC,43398,0,15,73.0548,0,10000921
-0,2012-10-06,Others,100847,0,16,77.0258,0,10000606
-0,2013-11-08,Others,164261,0,11,16.6959,0,10000136
-0,2012-04-13,FP-non GTC,3838,0,5,68.4347,0,10000451
-0,2012-07-03,FP-GTC,63861,0,-99,89.5962,0,10000893
-0,2012-02-21,Auction,40059,3,-99,84.7455,0,10000613
-0,2012-02-25,FP-non GTC,759,0,13,23.9258,0,10000826
-0,2013-10-30,FP-non GTC,57990,0,16,88.4859,0,10000782
-0,2013-10-12,FP-non GTC,57784,0,14,85.8986,0,10000220
-0,2012-08-15,FP-GTC,223,0,12,32.1974,0,10000316
-0,2012-10-18,FP-GTC,15115,0,16,52.5243,0,10000353
-0,2013-11-09,FP-GTC,20886,0,13,94.0128,0,10000690
-0,2012-09-27,ABIN,31673,0,12,24.1332,0,10000404
-0,2012-05-08,FP-GTC,95672,0,16,88.8853,0,10000984
-0,2013-01-14,Others,95173,0,12,86.3987,0,10000501
-0,2012-12-02,Auction,152801,0,-99,71.8799,0,10000021
-0,2012-03-13,ABIN,31519,3,15,96.8998,0,10000188
-0,2013-06-15,Auction,121153,0,14,66.2729,0,10000101
-0,2012-04-11,Others,63861,0,15,88.0097,0,10000536
-0,2013-01-25,FP-non GTC,1357,0,16,49.8991,0,10000637
-0,2013-10-17,FP-GTC,106246,0,15,92.2507,0,10000046
-0,2012-10-04,Others,4943,0,13,88.8498,0,10000215
-0,2013-05-14,Auction,314,0,13,77.3441,0,10000900
-0,2013-03-17,FP-non GTC,759,0,16,17.9719,0,10000759
-0,2013-01-20,FP-non GTC,51582,0,16,3.6644,0,10000961
-0,2013-03-20,FP-non GTC,57990,0,12,9.3680,0,10000846
-0,2012-06-06,FP-non GTC,165888,0,16,9.4584,0,10000312
-0,2012-12-13,Auction,43479,0,16,84.0632,0,10000349
-0,2012-05-13,FP-non GTC,139973,3,5,37.3510,0,10000790
-0,2013-03-27,Auction,48027,0,5,75.7167,0,10000424
-0,2013-08-08,FP-GTC,37831,0,-99,27.7800,0,10000637
-0,2012-04-06,ABIN,314,0,12,18.3274,0,10000749
-0,2013-10-11,FP-non GTC,57013,0,14,39.1868,0,10000599
-0,2012-01-18,FP-GTC,41940,0,12,94.5165,0,10000102
-0,2013-05-20,ABIN,50677,0,16,25.6136,0,10000423
-0,2012-06-24,FP-GTC,963,0,13,20.9442,0,10000069
-0,2013-04-01,ABIN,75665,0,12,65.3307,0,10000543
-0,2012-02-05,ABIN,50508,0,-99,69.4516,0,10000623
-0,2012-06-24,ABIN,314,0,5,13.8612,0,10000347
-0,2013-12-07,Others,1349,0,16,10.8619,0,10000784
-0,2013-03-01,ABIN,73506,0,11,7.7818,0,10000882
-0,2012-08-24,Auction,45333,0,16,94.9583,0,10000495
-0,2012-08-04,FP-non GTC,20886,0,15,69.7300,0,10000428
-0,2012-11-08,Auction,31519,0,15,41.4423,0,10000431
-0,2013-08-07,FP-GTC,156356,0,13,44.6853,0,10000305
-0,2013-09-25,ABIN,43398,0,16,46.5536,0,10000387
-0,2012-04-26,FP-non GTC,95672,0,13,30.9202,0,10000591
-0,2012-03-30,Others,159184,0,12,58.3993,0,10000381
-0,2012-04-25,Auction,73506,0,14,46.1501,0,10000010
-0,2012-09-13,FP-GTC,20485,0,5,2.4112,0,10000959
-0,2013-09-27,ABIN,80287,0,-99,64.4361,0,10000770
-0,2013-01-17,Others,36250,0,11,73.5773,0,10000948
-0,2012-07-07,Auction,103324,15,16,4.8559,0,10000123
-0,2013-08-15,ABIN,13836,0,16,25.8438,0,10000851
-0,2012-07-07,Others,12688,0,5,3.4934,0,10000096
-0,2012-06-03,Auction,80135,0,12,17.8258,0,10000144
-0,2013-12-24,FP-GTC,60340,0,11,22.6409,0,10000341
-0,2012-10-06,ABIN,36250,0,5,7.4322,0,10000384
-0,2013-01-03,Auction,175750,0,15,14.1976,0,10000366
-0,2013-04-15,Auction,88750,0,11,98.6057,0,10000584
-0,2013-06-03,FP-non GTC,1349,0,15,11.0419,0,10000866
-0,2013-01-30,FP-GTC,2635,0,14,54.5175,0,10000959
-0,2013-08-27,FP-non GTC,26249,0,13,50.0365,0,10000289
-0,2012-05-14,FP-non GTC,9426,3,5,60.0224,0,10000730
-0,2012-10-16,FP-non GTC,16145,0,-99,53.0896,0,10000195
-0,2012-09-24,FP-GTC,57013,0,14,81.7929,0,10000498
-0,2012-08-20,FP-non GTC,25147,0,-99,30.7605,0,10000242
-0,2012-02-19,Others,94847,0,13,20.6395,0,10000707
-0,2013-11-14,ABIN,50508,0,15,91.9217,0,10000208
-0,2012-07-10,Others,95672,0,13,21.1886,0,10000762
-0,2012-03-14,ABIN,44079,0,13,89.6838,0,10000227
-0,2012-01-17,FP-non GTC,314,0,14,44.3066,0,10000251
-0,2012-02-27,FP-non GTC,15868,0,14,80.8427,0,10000741
-0,2012-10-23,Others,1161,3,5,54.0104,0,10000289
-0,2013-06-12,Others,53064,0,12,87.1336,0,10000871
-0,2013-03-06,Auction,34273,100,15,6.7044,0,10000269
-0,2012-06-16,Others,533,0,14,43.7104,0,10000172
-0,2012-12-19,FP-GTC,15868,0,15,94.1727,0,10000366
-0,2012-07-21,ABIN,121153,0,12,42.3437,0,10000688
-0,2013-08-19,FP-non GTC,65,0,11,3.0275,0,10000055
-0,2012-08-08,Auction,15687,0,14,72.9505,0,10000725
-0,2013-12-13,FP-GTC,4943,0,-99,37.7781,0,10000932
-0,2013-05-18,FP-GTC,3838,0,14,45.1642,0,10000370
-0,2012-01-31,Auction,25147,0,14,78.6580,0,10000721
-0,2013-12-05,FP-GTC,62179,0,12,76.4692,0,10000423
-0,2012-03-07,Auction,175750,0,15,19.5357,0,10000053
-0,2013-09-30,Others,100847,0,12,38.2299,0,10000772
-0,2013-01-07,Others,15115,0,12,82.5099,0,10000864
-0,2013-11-04,FP-non GTC,57013,0,-99,61.3731,0,10000335
-0,2012-08-11,FP-non GTC,31673,0,12,12.7001,0,10000303
-0,2012-12-30,ABIN,156356,0,14,99.3562,0,10000427
-0,2013-05-19,Others,113593,15,11,31.1634,0,10000361
-0,2013-06-03,ABIN,314,0,13,32.6968,0,10000373
-0,2013-06-29,Auction,15808,15,12,18.3967,0,10000484
-0,2012-06-10,FP-GTC,61323,0,13,10.8143,0,10000403
-0,2012-01-29,FP-GTC,1349,0,15,0.4358,0,10000639
-0,2013-05-30,FP-non GTC,759,0,14,19.5920,0,10000116
-0,2013-07-28,ABIN,15868,0,14,59.7123,0,10000668
-0,2013-03-23,Auction,165888,0,12,67.1201,0,10000063
-0,2012-05-29,FP-GTC,118687,3,-99,98.2030,0,10000886
-0,2013-09-15,FP-non GTC,156614,0,5,43.1795,0,10000302
-0,2012-05-21,FP-GTC,223,0,-99,66.2850,0,10000118
-0,2012-04-03,FP-non GTC,26262,0,12,17.5241,0,10000440
-0,2012-05-22,ABIN,44079,0,16,25.2405,0,10000183
-0,2014-01-01,Others,50508,0,16,92.5843,0,10000911
-0,2012-04-08,Others,16145,3,12,90.2242,0,10000825
-0,2013-11-27,ABIN,95672,0,16,90.9380,0,10000829
-0,2013-03-22,Auction,41940,0,15,72.0648,0,10000795
-0,2012-04-06,Others,64076,0,14,86.9185,0,10000356
-0,2013-05-02,Others,156356,0,11,41.6871,0,10000122
-0,2012-04-02,Others,132939,0,12,75.8372,0,10000395
-0,2012-06-29,ABIN,26262,0,13,83.6043,0,10000120
-0,2012-07-06,FP-GTC,11554,0,16,38.4329,0,10000352
-0,2012-06-10,ABIN,156614,0,-99,94.1812,0,10000280
-0,2012-04-05,Auction,164261,0,12,0.9691,0,10000480
-0,2012-11-02,Auction,31519,0,11,98.0505,0,10000893
-0,2012-05-20,Auction,50508,0,15,61.9439,0,10000363
-0,2012-03-31,Auction,80135,0,16,11.1543,0,10000692
-0,2012-07-01,FP-non GTC,63889,0,-99,86.6557,0,10000402
-0,2013-01-04,ABIN,41940,0,-99,48.4505,0,10000296
-0,2013-02-10,FP-non GTC,63861,0,13,60.3535,0,10000214
-0,2012-03-25,Others,73506,0,11,74.2113,0,10000801
-0,2013-07-23,FP-non GTC,53064,0,16,69.2368,0,10000542
-0,2012-04-09,Auction,11848,0,15,76.1720,0,10000555
-0,2012-08-01,Others,12688,0,5,85.4589,0,10000130
-0,2012-07-15,FP-non GTC,31519,3,-99,17.8485,0,10000045
-0,2013-10-06,ABIN,1349,0,11,52.2780,0,10000622
-0,2012-10-26,Auction,25147,0,13,60.9912,0,10000379
-0,2013-01-04,FP-non GTC,37831,0,15,61.0134,0,10000785
-0,2012-03-24,Auction,139973,0,-99,8.1533,0,10000487
-0,2012-12-30,Auction,6762,0,-99,77.8179,0,10000645
-0,2012-10-03,ABIN,103178,15,12,15.2201,0,10000824
-0,2012-04-18,FP-non GTC,50508,0,14,11.9715,0,10000012
-0,2012-09-30,FP-non GTC,31673,0,12,90.0864,0,10000281
-0,2013-03-24,ABIN,4943,0,12,11.0294,0,10000785
-0,2012-05-02,Others,87118,0,14,52.6549,0,10000924
-0,2013-05-03,FP-GTC,24760,0,5,8.9077,0,10000758
-0,2012-08-05,FP-non GTC,38238,0,11,0.8463,0,10000066
-0,2012-08-15,Auction,67698,0,11,23.6278,0,10000085
-0,2013-04-04,Auction,164261,0,13,52.3718,0,10000965
-0,2013-04-18,ABIN,25147,0,14,71.7687,0,10000157
-0,2013-11-09,ABIN,43479,0,12,39.9269,0,10000024
-0,2013-05-02,FP-non GTC,175750,0,12,17.6149,0,10000862
-0,2012-09-01,ABIN,50508,0,14,99.1518,0,10000746
-0,2013-02-23,FP-non GTC,139255,15,-99,74.6182,0,10000447
-0,2012-01-19,FP-non GTC,6762,0,13,0.1297,0,10000189
-0,2012-11-10,Auction,113802,15,-99,11.9583,0,10000596
-0,2012-12-11,FP-non GTC,95173,0,5,67.9416,0,10000582
-0,2012-01-05,ABIN,2635,0,11,72.3723,0,10000375
-0,2012-05-04,FP-non GTC,43479,0,13,79.1307,0,10000944
-0,2013-07-20,ABIN,63861,0,13,86.7155,0,10000274
-0,2013-08-25,FP-GTC,80053,0,5,11.7108,0,10000059
-0,2012-10-24,ABIN,38238,0,5,79.2964,0,10000388
-0,2012-01-10,ABIN,73506,0,5,3.8878,0,10000895
-0,2013-07-11,Auction,174106,3,12,46.7087,0,10000683
-0,2013-06-16,FP-GTC,61323,0,15,64.0224,0,10000121
-0,2013-06-23,ABIN,63889,0,5,59.2766,0,10000466
-0,2013-01-25,Others,156356,0,14,79.9992,0,10000611
-0,2013-10-09,Auction,50508,0,16,70.9682,0,10000009
-0,2013-05-19,Auction,25147,0,16,8.0973,0,10000731
-0,2012-10-14,Auction,67698,0,5,49.0497,0,10000605
-0,2012-06-08,Auction,65,0,14,76.5149,0,10000165
-0,2012-12-21,FP-GTC,15868,0,5,45.9224,0,10000215
-0,2012-12-02,Auction,1357,0,15,8.9337,0,10000730
-0,2013-11-27,ABIN,24760,0,11,62.4675,0,10000932
-0,2013-11-24,Auction,95173,0,16,16.7927,0,10000795
-0,2013-11-30,ABIN,164,0,11,10.8897,0,10000914
-0,2012-12-22,Auction,15868,0,15,7.4153,0,10000540
-0,2012-05-13,FP-non GTC,26249,0,5,58.8647,0,10000898
-0,2013-06-22,ABIN,95672,3,12,71.8628,0,10000026
-0,2013-05-22,ABIN,63861,0,11,71.8836,0,10000650
-0,2012-11-17,FP-GTC,32876,0,12,35.6019,0,10000959
-0,2013-04-13,Auction,38238,0,5,21.7443,0,10000036
-0,2012-09-09,FP-non GTC,87118,0,14,46.2858,0,10000512
-0,2013-08-23,FP-non GTC,41940,0,11,98.4779,0,10000844
-0,2012-09-02,FP-non GTC,43398,0,12,91.8543,0,10000917
-0,2012-11-23,Others,145970,0,11,43.8631,0,10000997
-0,2012-01-22,FP-non GTC,164261,0,16,74.9772,0,10000091
-0,2012-05-02,ABIN,152801,0,13,15.2794,0,10000649
-0,2013-08-14,Others,50677,0,13,26.5329,0,10000275
-0,2012-06-04,Others,156614,0,14,42.7810,0,10000347
-0,2012-08-04,FP-non GTC,145970,0,15,92.6953,0,10000622
-0,2012-08-15,Auction,32876,0,16,35.2015,0,10000533
-0,2012-11-05,Auction,95672,0,11,0.5233,0,10000965
-0,2013-10-12,ABIN,165888,0,11,27.3564,0,10000036
-0,2012-10-22,ABIN,20485,0,14,28.6742,0,10000448
-0,2012-07-02,Auction,20213,0,15,81.3332,0,10000484
-0,2013-01-13,Others,152801,0,11,3.3406,0,10000124
-0,2012-05-17,Others,132939,0,15,55.3503,0,10000086
-0,2013-02-05,Others,48904,0,15,25.2340,0,10000246
-0,2013-09-13,FP-GTC,32876,0,-99,10.7672,0,10000729
-0,2012-06-14,ABIN,20865,0,14,47.9841,0,10000406
-0,2013-02-11,Auction,152801,0,5,86.4355,0,10000050
-0,2012-12-13,Auction,50508,0,5,72.5114,0,10000715
-0,2013-11-30,Auction,3838,0,-99,6.8517,0,10000706
-0,2013-03-15,FP-non GTC,20865,0,14,93.3040,0,10000926
-0,2012-08-12,FP-GTC,57784,0,-99,18.4045,0,10000231
-0,2013-10-19,Others,1161,3,14,79.1353,0,10000561
-0,2013-02-17,Others,31519,0,-99,49.0409,0,10000904
-0,2012-03-30,FP-non GTC,314,0,11,93.6413,0,10000041
-0,2012-08-26,Others,175750,0,-99,46.6557,0,10000719
-0,2012-12-26,Auction,139973,0,15,6.9809,0,10000044
-0,2013-01-30,Others,159184,0,13,45.4599,0,10000131
-0,2012-04-27,Auction,48904,0,5,16.3472,0,10000157
-0,2012-11-10,ABIN,16145,0,13,0.4693,0,10000668
-0,2012-05-13,Auction,121153,0,5,24.6420,0,10000314
-0,2012-02-01,FP-GTC,6762,0,13,5.9417,0,10000851
-0,2012-07-16,Auction,95672,0,-99,11.9934,0,10000828
-0,2012-09-21,Auction,106246,0,5,39.0057,0,10000826
-0,2012-10-20,FP-GTC,314,0,16,5.5339,0,10000011
-0,2013-04-29,FP-non GTC,100847,0,13,3.3872,0,10000840
-0,2013-02-16,FP-non GTC,43398,0,13,3.4171,0,10000106
-0,2013-09-11,FP-GTC,63889,0,12,15.4662,0,10000004
-0,2012-12-18,Others,80287,0,15,71.3090,0,10000722
-0,2013-09-14,FP-non GTC,20865,0,-99,98.0987,0,10000591
-0,2013-01-27,FP-non GTC,57990,3,14,84.9483,0,10000304
-0,2013-02-07,Others,95672,0,14,9.7034,0,10000894
-0,2013-03-24,FP-non GTC,38238,0,-99,98.5328,0,10000286
-0,2012-01-05,Others,164,0,15,21.3336,0,10000151
-0,2012-11-14,Auction,57990,0,12,91.1773,0,10000087
-0,2013-03-24,Auction,13987,0,13,5.9953,0,10000551
-0,2013-12-26,FP-GTC,32876,0,11,86.1904,0,10000012
-0,2013-06-23,FP-non GTC,1504,0,12,9.4689,0,10000720
-0,2012-01-11,FP-GTC,57013,0,13,96.6705,0,10000790
-0,2012-08-30,FP-non GTC,62179,0,5,22.4244,0,10000403
-0,2012-09-01,ABIN,61323,0,12,30.1513,0,10000283
-0,2013-04-20,Auction,16145,3,12,77.9327,0,10000043
-0,2013-10-03,Others,963,0,14,82.3371,0,10000002
-0,2012-10-23,ABIN,45333,0,16,86.7017,0,10000180
-0,2013-10-09,ABIN,63889,0,-99,33.9942,0,10000485
-0,2013-04-28,ABIN,73506,0,5,22.6563,0,10000070
-0,2013-07-07,ABIN,57990,0,11,11.7659,0,10000834
-0,2012-01-04,FP-non GTC,165888,0,16,78.7873,0,10000743
-0,2012-05-05,ABIN,158666,15,16,55.1036,0,10000372
-0,2012-08-04,ABIN,43479,0,13,61.3469,0,10000425
-0,2013-02-18,Others,161567,15,5,43.5109,0,10000315
-0,2013-08-09,FP-GTC,161567,15,14,0.9756,0,10000420
-0,2012-04-30,Others,65,0,12,31.5741,0,10000858
-0,2013-10-16,FP-non GTC,16509,0,15,8.4323,0,10000376
-0,2013-11-06,FP-GTC,15115,0,12,91.7204,0,10000495
-0,2012-04-13,ABIN,37831,0,16,86.9150,0,10000947
-0,2013-04-21,FP-GTC,95672,0,-99,60.6048,0,10000052
-0,2013-12-04,FP-GTC,121153,0,13,9.3639,0,10000384
-0,2012-08-03,Others,50508,0,13,74.2474,0,10000103
-0,2013-12-29,FP-GTC,1357,0,5,46.9382,0,10000987
-0,2012-11-01,Others,46575,0,13,54.4520,0,10000247
-0,2012-06-30,FP-non GTC,26249,0,5,74.2755,0,10000641
-0,2012-06-20,Auction,26262,0,11,11.9173,0,10000418
-0,2012-07-24,Others,80287,0,14,49.7794,0,10000590
-0,2012-08-11,FP-GTC,26262,0,11,16.2910,0,10000698
-0,2013-06-27,ABIN,63861,0,-99,5.4086,0,10000810
-0,2012-04-28,FP-GTC,15115,0,15,91.7724,0,10000414
-0,2013-04-19,ABIN,73506,0,13,14.8212,0,10000728
-0,2012-01-25,Others,32876,0,13,23.4295,0,10000673
-0,2013-11-06,ABIN,139973,3,12,41.9681,0,10000902
-0,2013-03-09,Auction,61323,0,15,53.8594,0,10000323
-0,2012-03-01,ABIN,60340,0,11,78.1279,0,10000947
-0,2012-07-28,FP-non GTC,279,15,16,80.8961,0,10000691
-0,2013-02-04,Others,24541,0,11,40.4673,0,10000727
-0,2012-02-24,Auction,152801,0,13,9.8921,0,10000661
-0,2013-08-13,ABIN,156614,0,15,61.6452,0,10000734
-0,2012-04-02,ABIN,63861,3,11,18.4644,0,10000446
-0,2013-09-28,Others,95173,0,-99,51.2096,0,10000049
-0,2013-08-13,FP-non GTC,64076,0,12,85.3952,0,10000581
-0,2012-09-09,ABIN,20485,0,15,0.5979,0,10000081
-0,2012-04-26,Auction,145970,0,11,40.5874,0,10000401
-0,2013-05-10,Others,26262,0,11,83.8561,0,10000545
-0,2013-05-23,FP-GTC,80287,0,11,27.6123,0,10000529
-0,2012-01-08,Others,175750,0,14,45.4268,0,10000408
-0,2012-07-20,ABIN,38238,0,16,44.1376,0,10000796
-0,2012-09-15,Auction,65,0,-99,97.3146,0,10000792
-0,2012-10-17,ABIN,65,0,12,68.5363,0,10000569
-0,2013-08-03,FP-GTC,40059,3,15,87.3892,0,10000317
-0,2013-08-28,FP-non GTC,80287,0,14,61.9829,0,10000015
-0,2012-07-21,Auction,25147,0,11,24.7662,0,10000333
-0,2013-10-17,Auction,132939,0,11,11.6038,0,10000926
-0,2012-09-04,Auction,44079,0,15,75.1095,0,10000435
-0,2012-11-01,Auction,67698,0,5,8.8273,0,10000834
-0,2013-12-02,Auction,24541,0,15,95.2195,0,10000259
-0,2012-02-06,Auction,533,0,12,66.4962,0,10000007
-0,2013-12-03,FP-non GTC,99985,0,12,0.0537,0,10000244
-0,2013-05-11,FP-non GTC,60606,3,12,76.6828,0,10000002
-0,2012-09-18,Auction,156614,0,16,72.1912,0,10000174
-0,2012-05-19,Auction,60340,0,16,42.9975,0,10000813
-0,2012-04-26,FP-GTC,165888,0,14,88.3875,0,10000919
-0,2013-08-30,Others,963,0,15,6.6424,0,10000994
-0,2013-08-28,ABIN,10866,0,12,79.7295,0,10000832
-0,2012-01-16,ABIN,145970,0,11,72.3940,0,10000008
-0,2013-12-23,Others,65,0,15,94.0397,0,10000812
-0,2013-09-06,FP-GTC,25147,0,11,72.9395,0,10000956
-0,2012-05-07,FP-GTC,16509,0,11,9.8290,0,10000940
-0,2013-09-07,Others,139973,0,11,22.9964,0,10000563
-0,2012-11-17,Others,65,0,-99,62.7316,0,10000227
-0,2012-05-14,ABIN,57013,0,13,19.0969,0,10000844
-0,2012-06-01,FP-non GTC,30059,3,16,26.9105,0,10000516
-0,2012-11-24,Auction,75665,0,15,26.5788,0,10000717
-0,2013-10-23,FP-non GTC,75708,3,12,10.5298,0,10000238
-0,2012-10-19,Auction,64076,0,11,49.0746,0,10000389
-0,2012-02-17,ABIN,65,0,12,58.3004,0,10000219
-0,2012-01-08,FP-GTC,15115,0,16,3.2380,0,10000435
-0,2012-02-04,Others,164261,0,14,45.9304,0,10000748
-0,2012-08-25,FP-GTC,156356,0,5,70.3654,0,10000864
-0,2013-04-10,FP-GTC,38238,0,5,72.9872,0,10000638
-0,2013-05-16,FP-GTC,175750,0,5,95.3008,0,10000366
-0,2012-06-27,FP-GTC,100847,0,14,64.8447,0,10000686
-0,2013-03-06,Auction,139255,15,12,20.7811,0,10000261
-0,2013-08-04,Auction,67698,0,14,20.1344,0,10000142
-0,2013-08-15,Auction,20865,0,12,64.6842,0,10000235
-0,2012-04-17,Auction,24760,0,-99,88.4159,0,10000039
-0,2013-03-14,Others,75665,0,13,36.9770,0,10000480
-0,2012-12-09,Others,139973,0,15,85.1056,0,10000801
-0,2012-07-02,Others,36250,0,11,71.5548,0,10000926
-0,2013-08-04,FP-non GTC,31519,0,16,23.9847,0,10000680
-0,2013-02-15,FP-non GTC,80053,0,15,68.4438,0,10000487
-0,2012-07-12,FP-non GTC,64076,0,5,54.5740,0,10000628
-0,2012-11-07,Others,164,0,12,45.1637,0,10000846
-0,2013-09-14,FP-GTC,106340,15,15,38.1572,0,10000201
-0,2013-12-19,Auction,759,0,-99,53.0086,0,10000250
-0,2012-05-11,Others,16509,0,-99,68.3043,0,10000809
-0,2013-07-20,Others,45333,0,15,58.5554,0,10000981
-0,2013-03-24,FP-GTC,20485,0,5,62.1012,0,10000877
-0,2013-08-09,ABIN,63861,0,5,44.1975,0,10000554
-0,2013-12-04,FP-GTC,53064,0,14,29.8358,0,10000491
-0,2013-07-06,ABIN,139973,0,5,26.8986,0,10000978
-0,2013-04-29,FP-non GTC,67698,0,11,46.9203,0,10000148
-0,2013-10-18,FP-non GTC,95173,0,14,25.5313,0,10000068
-0,2014-01-01,Auction,6762,0,13,30.6746,0,10000592
-0,2013-07-14,Auction,13836,0,16,79.4600,0,10000944
-0,2012-10-12,FP-GTC,145970,0,5,32.3728,0,10000396
-0,2013-07-28,Auction,1357,0,13,8.7896,0,10000836
-0,2013-12-06,FP-non GTC,73506,0,-99,95.4405,0,10000914
-0,2012-08-01,FP-non GTC,37831,0,11,12.8673,0,10000993
-0,2013-09-07,FP-non GTC,223,0,-99,16.9111,0,10000028
-0,2012-12-27,FP-GTC,82494,15,15,73.5475,0,10000609
-0,2012-04-07,Others,36250,0,16,7.5886,0,10000838
-0,2012-10-07,FP-GTC,139973,0,12,45.3171,0,10000439
-0,2012-01-19,FP-non GTC,63861,0,16,31.1155,0,10000338
-0,2013-12-16,FP-non GTC,164,0,16,41.0688,0,10000377
-0,2012-03-25,ABIN,41940,0,14,8.1251,0,10000318
-0,2012-04-15,FP-non GTC,65,0,-99,56.9993,0,10000697
-0,2012-11-27,Auction,25147,0,14,14.5707,0,10000303
-0,2012-08-04,FP-non GTC,20213,0,11,20.6361,0,10000443
-0,2013-11-15,Auction,95672,3,5,40.1888,0,10000506
-0,2013-12-26,Others,16145,0,13,34.4427,0,10000307
-0,2013-11-25,Others,139973,0,16,60.8826,0,10000762
-0,2013-09-01,Others,24760,0,16,22.9159,0,10000759
-0,2012-12-10,ABIN,53064,0,14,51.2661,0,10000626
-0,2013-08-24,Auction,759,0,13,48.4301,0,10000370
-0,2012-11-29,Auction,32876,0,5,86.2750,0,10000755
-0,2012-06-11,Others,61323,0,13,14.7483,0,10000812
-0,2012-04-23,Others,67698,0,-99,21.2161,0,10000384
-0,2013-07-12,Others,40059,3,5,88.2983,0,10000937
-0,2012-12-27,FP-GTC,13987,0,5,29.1748,0,10000223
-0,2013-02-15,Others,113802,15,5,31.4243,0,10000672
-0,2013-02-20,Auction,166013,15,15,22.9952,0,10000618
-0,2013-03-13,ABIN,73506,0,16,94.7440,0,10000476
-0,2012-02-19,FP-GTC,95672,0,15,28.6221,0,10000421
-0,2013-11-30,FP-GTC,166013,15,5,92.5521,0,10000729
-0,2012-09-30,ABIN,36250,0,15,18.6346,0,10000606
-0,2013-02-13,Auction,20886,0,14,24.7987,0,10000535
-0,2013-11-10,FP-GTC,759,0,13,53.7935,0,10000711
-0,2013-02-10,Auction,20886,0,12,84.2953,0,10000811
-0,2012-05-28,ABIN,10866,0,13,64.3564,0,10000804
-0,2013-03-19,Auction,314,0,16,36.1791,0,10000379
-0,2012-01-26,FP-non GTC,132939,0,-99,31.4514,0,10000690
-0,2013-02-02,FP-non GTC,20485,0,14,97.4265,0,10000642
-0,2013-07-25,Auction,139973,0,16,12.3287,0,10000972
-0,2012-09-21,Others,15115,0,11,2.7636,0,10000896
-0,2012-07-26,ABIN,533,0,15,0.6745,0,10000762
-0,2012-06-02,FP-non GTC,48904,0,-99,56.1505,0,10000937
-0,2012-12-11,FP-non GTC,155226,0,14,59.4547,0,10000197
-0,2012-05-30,Others,26249,0,14,18.0992,0,10000836
-0,2012-12-25,Auction,20485,0,15,23.8257,0,10000992
-0,2013-07-08,ABIN,57013,0,12,59.6550,0,10000196
-0,2013-02-22,Auction,156356,0,14,36.1970,0,10000014
-0,2012-03-28,ABIN,25147,0,-99,56.3283,0,10000943
-0,2012-10-09,Others,63861,0,16,9.5503,0,10000659
-0,2012-12-29,FP-non GTC,95672,0,12,57.7305,0,10000568
-0,2013-06-14,Others,80053,0,13,89.8029,0,10000580
-0,2012-02-14,Others,67698,0,16,73.0734,0,10000857
-0,2013-05-14,FP-non GTC,62179,0,5,58.7369,0,10000091
-0,2013-06-16,FP-GTC,1504,0,5,7.2913,0,10000054
-0,2013-07-30,FP-non GTC,31673,0,-99,13.2961,0,10000654
-0,2013-05-22,ABIN,130,0,-99,1.3005,0,10000925
-0,2012-03-06,FP-GTC,35570,100,15,45.7240,0,10000773
-0,2012-07-20,Auction,15868,0,15,88.4248,0,10000965
-0,2013-10-02,Auction,38238,0,15,34.7237,0,10000987
-0,2012-11-06,Others,67698,0,-99,3.3750,0,10000902
-0,2012-01-11,FP-GTC,62179,0,11,77.9520,0,10000540
-0,2012-09-23,FP-GTC,64076,0,12,57.1896,0,10000772
-0,2012-03-03,ABIN,103178,15,11,26.3886,0,10000000
-0,2012-05-26,FP-GTC,48027,0,16,87.2722,0,10000836
-0,2012-05-06,ABIN,38238,0,16,34.7669,0,10000942
-0,2014-01-01,Auction,170302,15,11,62.2724,0,10000988
-0,2012-06-19,FP-GTC,37831,0,13,78.1533,0,10000362
-0,2012-01-12,ABIN,48904,0,-99,80.2518,0,10000266
-0,2013-06-24,ABIN,10866,0,12,87.3794,0,10000655
-0,2013-07-17,Others,1349,0,12,40.6979,0,10000754
-0,2013-06-14,Auction,139973,0,14,54.9050,0,10000105
-0,2012-06-18,ABIN,15868,0,-99,69.2751,0,10000988
-0,2012-09-07,Auction,1349,0,11,24.7687,0,10000323
-0,2012-02-01,ABIN,61323,0,-99,72.0754,0,10000321
-0,2013-07-28,ABIN,152801,0,11,71.8684,0,10000701
-0,2013-04-16,Others,60340,0,-99,9.8364,0,10000952
-0,2012-01-06,ABIN,80053,0,12,45.9154,0,10000726
-0,2012-02-07,Auction,223,0,12,90.7522,0,10000949
-0,2012-09-15,Others,50677,0,5,50.2863,0,10000285
-0,2013-04-01,Auction,75665,0,12,40.7265,0,10000498
-0,2012-11-09,ABIN,95672,0,12,85.7492,0,10000180
-0,2013-09-22,FP-GTC,16145,0,16,37.4930,0,10000453
-0,2012-11-02,Auction,73506,0,5,61.7567,0,10000361
-0,2013-10-27,Auction,963,0,-99,26.4725,0,10000909
-0,2012-01-13,FP-GTC,10866,0,16,84.3734,0,10000539
-0,2012-05-26,FP-non GTC,152801,0,5,77.1831,0,10000961
-0,2013-06-04,ABIN,57013,0,-99,59.7825,0,10000542
-0,2012-11-26,FP-GTC,9426,3,5,83.8080,0,10000847
-0,2013-12-11,ABIN,67698,0,14,67.3561,0,10000409
-0,2013-06-01,ABIN,95173,0,13,50.4721,0,10000245
-0,2012-07-25,FP-GTC,175750,0,16,62.5123,0,10000399
-0,2012-01-22,FP-GTC,31673,0,13,8.1462,0,10000125
-0,2012-01-17,Auction,63861,0,15,86.7921,0,10000067
-0,2012-07-06,Others,20886,0,-99,91.8578,0,10000349
-0,2012-02-26,FP-non GTC,216,0,13,25.5255,0,10000465
-0,2013-10-05,Others,65,0,11,50.8591,0,10000718
-0,2012-12-30,FP-non GTC,87118,0,11,81.3002,0,10000122
-0,2013-08-20,ABIN,130,0,13,61.5051,0,10000685
-0,2012-11-07,ABIN,57013,0,15,87.1409,0,10000157
-0,2012-11-26,Others,75665,0,16,85.4010,0,10000935
-0,2013-12-17,FP-GTC,63861,0,14,45.7096,0,10000650
-0,2012-08-12,FP-non GTC,20485,0,-99,44.4235,0,10000954
-0,2012-01-17,Auction,314,0,12,75.8335,0,10000514
-0,2013-01-24,ABIN,2635,0,12,10.6521,0,10000839
-0,2012-02-05,ABIN,3838,0,12,76.2544,0,10000344
-0,2013-05-13,FP-GTC,106246,0,14,25.5924,0,10000291
-0,2013-03-16,Auction,82494,15,5,15.6450,0,10000366
-0,2012-01-06,Auction,65,0,11,9.8382,0,10000415
-0,2012-08-10,ABIN,24541,0,-99,21.1252,0,10000872
-0,2012-06-01,Others,106246,0,11,51.6742,0,10000000
-0,2013-05-10,Others,63861,0,-99,69.1514,0,10000869
-0,2013-04-04,FP-non GTC,73506,0,13,7.8706,0,10000986
-0,2013-10-23,ABIN,43972,100,-99,31.5134,0,10000011
-0,2013-05-10,Auction,80053,0,-99,2.8786,0,10000544
-0,2012-03-26,FP-non GTC,80053,0,13,59.3087,0,10000881
-0,2012-07-24,Auction,20485,0,-99,49.9084,0,10000757
-0,2013-04-10,ABIN,66767,15,5,48.4303,0,10000471
-0,2013-07-09,ABIN,13987,0,12,82.6598,0,10000495
-0,2012-12-25,ABIN,145970,0,5,6.4088,0,10000723
-0,2013-11-14,FP-non GTC,113593,15,15,10.9465,0,10000701
-0,2013-08-17,Auction,26249,0,12,90.2925,0,10000684
-0,2013-03-28,Auction,53064,0,13,1.5558,0,10000884
-0,2013-05-04,ABIN,88750,0,15,41.8885,0,10000078
-0,2013-07-20,Others,60340,0,-99,79.8564,0,10000590
-0,2012-12-10,ABIN,1504,0,13,22.2111,0,10000603
-0,2012-01-18,FP-non GTC,1349,0,-99,40.5209,0,10000684
-0,2013-01-21,FP-non GTC,25147,0,13,47.2094,0,10000676
-0,2013-10-18,FP-non GTC,164261,0,5,71.2879,0,10000998
-0,2012-08-18,Others,61323,0,15,59.1738,0,10000705
-0,2012-01-28,ABIN,1357,0,11,31.3101,0,10000131
-0,2013-12-25,Others,94847,0,13,90.8027,0,10000674
-0,2013-12-26,Others,156614,0,5,14.9093,0,10000123
-0,2012-06-19,Others,43398,0,13,85.5022,0,10000874
-0,2012-12-12,FP-non GTC,145970,0,16,71.8714,0,10000686
-0,2012-02-11,Others,24760,0,14,49.0051,0,10000611
-0,2013-06-02,Others,759,0,13,56.3362,0,10000185
-0,2012-09-30,Auction,75665,0,16,89.9688,0,10000378
-0,2013-11-10,Others,43479,0,16,2.0896,0,10000272
-0,2012-06-19,FP-GTC,759,0,5,34.9300,0,10000531
-0,2012-02-02,FP-non GTC,57990,3,-99,12.9845,0,10000591
-0,2013-04-04,FP-GTC,73506,0,11,32.9647,0,10000464
-0,2012-07-01,FP-non GTC,24760,0,11,84.6443,0,10000646
-0,2012-10-29,ABIN,46575,0,5,71.8566,0,10000247
-0,2013-11-19,Others,25147,0,16,61.2794,0,10000146
-0,2013-09-10,Others,95672,0,13,13.8226,0,10000353
-0,2013-07-29,FP-non GTC,67698,0,16,36.8872,0,10000266
-0,2013-10-17,ABIN,73506,0,11,57.0211,0,10000341
-0,2013-03-25,Others,20213,0,11,55.2006,0,10000627
-0,2012-05-30,FP-GTC,38238,0,5,9.9162,0,10000418
-0,2013-07-17,Auction,100847,0,11,11.3573,0,10000097
-0,2012-02-11,Others,166013,15,16,49.2212,0,10000634
-0,2012-08-17,FP-GTC,45333,0,16,99.5103,0,10000769
-0,2012-03-26,Auction,11554,0,14,58.2225,0,10000444
-0,2012-11-16,FP-non GTC,100847,0,16,67.4826,0,10000523
-0,2012-06-22,Auction,20213,0,5,44.8452,0,10000639
-0,2012-07-04,ABIN,1504,0,16,64.0222,0,10000827
-0,2012-10-10,Others,16145,0,14,81.1907,0,10000468
-0,2012-06-12,FP-non GTC,45333,0,14,20.5268,0,10000808
-0,2012-07-27,FP-non GTC,156614,0,11,28.1814,0,10000053
-0,2012-05-01,ABIN,61323,0,12,78.4665,0,10000767
-0,2013-05-04,ABIN,20485,0,16,94.0624,0,10000715
-0,2013-06-21,ABIN,20485,0,-99,92.7954,0,10000066
-0,2013-09-23,FP-non GTC,25147,0,5,81.3970,0,10000728
-0,2013-01-13,Auction,88750,0,-99,18.9880,0,10000265
-0,2012-01-12,FP-GTC,2023,0,14,91.3572,0,10000139
-0,2013-11-08,Auction,63861,0,11,17.1466,0,10000247
-0,2012-12-02,Auction,87118,0,5,75.0804,0,10000364
-0,2013-09-23,Others,46575,0,11,10.2322,0,10000384
-0,2012-01-26,FP-GTC,20865,0,16,6.7215,0,10000538
-0,2013-09-11,Others,158666,15,14,10.9434,0,10000153
-0,2013-02-19,Others,106340,15,16,32.2718,0,10000481
-0,2012-08-21,FP-non GTC,57784,0,11,29.4936,0,10000682
-0,2013-12-17,ABIN,314,0,13,45.6416,0,10000323
-0,2012-11-28,ABIN,46575,0,11,74.3051,0,10000167
-0,2013-10-25,Auction,1357,0,16,63.2572,0,10000556
-0,2012-08-13,ABIN,51582,0,16,12.5932,0,10000744
-0,2012-03-14,FP-GTC,1161,3,15,36.8516,0,10000221
-0,2012-10-12,Others,1349,0,13,9.8277,0,10000954
-0,2013-07-30,Auction,16145,0,13,25.6803,0,10000527
-0,2013-04-20,Auction,121153,0,-99,94.4016,0,10000033
-0,2013-12-11,FP-GTC,45333,0,13,50.3231,0,10000488
-0,2012-03-12,Others,80135,0,14,46.9661,0,10000800
-0,2012-01-26,FP-non GTC,62179,0,15,6.7796,0,10000298
-0,2012-07-18,Others,13836,0,12,59.3458,0,10000932
-0,2013-11-01,Others,13836,0,13,7.5394,0,10000014
-0,2013-06-11,Auction,16509,0,12,53.2002,0,10000309
-0,2012-01-14,Auction,20865,0,12,9.5903,0,10000003
-0,2013-12-13,FP-GTC,43479,0,5,28.2234,0,10000654
-0,2012-02-17,Auction,6762,0,5,84.1040,0,10000246
-0,2013-11-01,FP-GTC,16145,0,5,44.8790,0,10000358
-0,2013-08-26,Others,67698,0,5,24.7653,0,10000110
-0,2012-02-01,FP-non GTC,99985,0,5,73.9172,0,10000822
-0,2013-01-05,Auction,11554,0,13,82.7057,0,10000554
-0,2012-12-05,FP-GTC,13987,0,-99,91.9448,0,10000377
-0,2012-04-27,FP-non GTC,13836,0,14,93.1923,0,10000766
-0,2012-09-23,ABIN,43479,0,16,74.2753,0,10000603
-0,2013-11-21,ABIN,164,0,14,81.0875,0,10000489
-0,2013-04-04,Auction,170083,3,12,12.4107,0,10000183
-0,2013-02-02,Auction,30059,3,11,66.0915,0,10000644
-0,2013-09-29,Auction,2635,0,13,41.5935,0,10000488
-0,2012-11-27,ABIN,26262,0,16,27.6081,0,10000191
-0,2013-01-10,Others,130,0,11,19.0647,0,10000056
-0,2013-07-28,FP-non GTC,164261,0,5,25.9751,0,10000663
-0,2012-10-05,FP-GTC,10866,0,-99,42.0352,0,10000058
-0,2013-11-22,Auction,20485,0,16,18.9452,0,10000996
-0,2013-11-30,FP-non GTC,175750,3,11,10.0374,0,10000358
-0,2013-04-23,ABIN,113802,15,15,40.0768,0,10000497
-0,2012-09-17,FP-GTC,216,0,14,32.8424,0,10000113
-0,2012-11-02,FP-non GTC,16145,3,14,63.9673,0,10000471
-0,2013-06-27,ABIN,67698,0,16,35.2993,0,10000722
-0,2013-12-16,ABIN,121153,0,-99,60.6539,0,10000709
-0,2012-03-27,ABIN,20886,0,12,69.1836,0,10000768
-0,2013-11-04,ABIN,57013,0,14,37.7805,0,10000450
-0,2012-09-04,FP-non GTC,57990,0,13,13.8693,0,10000743
-0,2013-06-22,Auction,20485,0,16,17.1755,0,10000336
-0,2012-10-14,FP-GTC,156614,0,12,29.0308,0,10000577
-0,2013-08-04,ABIN,106246,0,13,98.4005,0,10000297
-0,2012-06-13,ABIN,43398,0,16,95.3892,0,10000984
-0,2012-05-21,Auction,20485,0,12,44.0443,0,10000357
-0,2013-08-12,Auction,165888,0,14,0.5521,0,10000804
-0,2013-03-12,Others,95672,0,14,54.3816,0,10000338
-0,2012-11-29,Others,15868,0,11,9.4996,0,10000177
-0,2013-10-25,FP-GTC,31519,0,12,58.8241,0,10000720
-0,2012-03-08,FP-GTC,165888,0,16,46.6778,0,10000484
-0,2012-04-30,Auction,1504,0,11,48.3609,0,10000835
-0,2012-08-31,FP-GTC,314,0,15,41.9195,0,10000284
-0,2012-04-02,Auction,33038,15,13,89.7577,0,10000947
-0,2013-07-21,Auction,45333,0,16,0.4349,0,10000941
-0,2012-07-13,ABIN,16509,0,5,98.1480,0,10000363
-0,2013-06-05,ABIN,4943,0,5,96.3653,0,10000029
-0,2013-10-12,FP-GTC,15115,0,12,64.1499,0,10000339
-0,2012-11-08,FP-GTC,25147,0,15,87.9610,0,10000149
-0,2013-02-17,Others,61323,0,16,92.5311,0,10000932
-0,2012-04-26,Others,20485,0,11,20.2247,0,10000245
-0,2013-07-13,FP-non GTC,95173,0,16,95.5853,0,10000150
-0,2012-10-08,FP-GTC,11554,0,15,29.6721,0,10000584
-0,2012-01-05,ABIN,31519,0,5,40.7845,0,10000365
-0,2013-07-11,ABIN,139973,0,16,64.1171,0,10000712
-0,2012-02-24,FP-GTC,62179,0,-99,93.5801,0,10000634
-0,2012-07-29,Others,152801,0,-99,23.4592,0,10000351
-0,2012-08-19,Others,10866,0,5,90.3864,0,10000665
-0,2012-05-10,ABIN,174053,3,14,90.5710,0,10000685
-0,2013-11-10,Auction,46575,0,-99,86.3150,0,10000982
-0,2012-05-03,FP-GTC,67703,3,14,17.5203,0,10000985
-0,2013-04-13,Auction,20213,0,-99,28.1159,0,10000357
-0,2013-12-01,FP-non GTC,13987,0,5,75.0139,0,10000703
-0,2013-06-06,Auction,10866,0,5,57.2098,0,10000916
-0,2013-04-20,Others,63861,0,15,97.3023,0,10000021
-0,2012-02-01,FP-non GTC,13836,0,13,54.2758,0,10000217
-0,2013-11-01,FP-non GTC,99985,0,12,76.3203,0,10000817
-0,2013-02-23,Others,80287,0,13,17.8778,0,10000937
-0,2012-04-22,Auction,3838,0,14,64.5441,0,10000466
-0,2013-03-14,Others,20485,0,11,25.2019,0,10000098
-0,2012-04-01,Others,31519,3,15,65.6555,0,10000121
-0,2012-01-28,FP-GTC,60606,3,14,11.4888,0,10000057
-0,2012-12-23,Auction,31519,0,14,38.8159,0,10000563
-0,2013-02-12,ABIN,4943,0,14,99.1697,0,10000525
-0,2013-08-07,Auction,67698,0,-99,42.5442,0,10000547
-0,2012-12-21,FP-non GTC,57990,0,11,56.5816,0,10000092
-0,2013-02-28,FP-GTC,20485,0,14,13.4842,0,10000463
-0,2013-09-26,ABIN,10866,0,16,28.3103,0,10000185
-0,2013-05-16,FP-GTC,63889,0,-99,16.0623,0,10000533
-0,2012-04-03,ABIN,32876,0,11,88.8682,0,10000702
-0,2013-02-28,FP-GTC,50508,0,-99,80.6989,0,10000669
-0,2012-07-24,ABIN,95672,0,12,77.5250,0,10000322
-0,2013-07-21,Auction,31519,0,11,26.2578,0,10000442
-0,2012-07-18,Others,43398,0,11,16.4674,0,10000831
-0,2013-07-31,Auction,26262,0,11,60.5816,0,10000182
-0,2013-12-18,ABIN,164261,0,14,61.6805,0,10000522
-0,2013-03-31,FP-GTC,25147,0,13,75.9632,0,10000685
-0,2012-05-30,FP-GTC,15687,0,13,61.0666,0,10000600
-0,2012-04-15,FP-non GTC,57784,0,5,30.0380,0,10000419
-0,2013-05-19,ABIN,139973,3,15,15.0669,0,10000388
-0,2013-02-20,Others,164261,0,15,18.6774,0,10000276
-0,2012-03-16,FP-non GTC,61323,0,14,49.2798,0,10000187
-0,2013-11-22,FP-GTC,62179,0,-99,92.8435,0,10000014
-0,2013-08-25,ABIN,1504,0,-99,39.5945,0,10000183
-0,2013-08-14,FP-non GTC,1357,0,-99,47.7304,0,10000079
-0,2012-12-19,Others,32876,0,16,67.1409,0,10000867
-0,2012-03-28,Others,80135,0,16,62.6308,0,10000577
-0,2013-01-01,FP-GTC,38238,0,5,52.0108,0,10000514
-0,2012-02-28,Auction,63889,0,15,28.0304,0,10000595
-0,2013-08-04,Auction,61323,0,16,38.9078,0,10000608
-0,2012-06-21,FP-GTC,95173,0,15,94.9831,0,10000156
-0,2013-06-06,Auction,57013,0,11,75.3949,0,10000992
-0,2012-08-08,FP-non GTC,26249,0,11,64.1580,0,10000469
-0,2012-05-13,ABIN,132939,0,-99,71.6430,0,10000542
-0,2012-03-06,FP-non GTC,57990,0,11,77.1134,0,10000093
-0,2012-08-07,ABIN,16145,3,16,65.5825,0,10000841
-0,2012-07-29,FP-GTC,164,0,5,62.5545,0,10000552
-0,2012-07-26,FP-non GTC,37831,0,13,2.1842,0,10000802
-0,2013-11-23,ABIN,13987,0,11,9.7841,0,10000094
-0,2013-12-13,Auction,1349,0,11,93.3393,0,10000024
-0,2013-09-25,ABIN,80053,0,13,0.2014,0,10000693
-0,2012-05-17,ABIN,31519,0,16,39.1773,0,10000060
-0,2013-02-27,ABIN,175750,0,15,44.1286,0,10000721
-0,2012-04-04,Others,1349,0,16,45.4831,0,10000828
-0,2012-04-12,Auction,63889,0,-99,84.5028,0,10000402
-0,2013-07-02,FP-non GTC,80053,0,5,44.1441,0,10000742
-0,2013-12-22,Others,1349,0,13,83.7579,0,10000223
-0,2013-06-10,ABIN,95173,0,15,20.0108,0,10000952
-0,2012-10-20,Auction,11848,0,13,33.2825,0,10000329
-0,2013-10-17,ABIN,26249,0,16,18.9369,0,10000685
-0,2012-08-20,FP-non GTC,3838,0,-99,35.9469,0,10000069
-0,2012-02-12,FP-GTC,31519,3,14,35.2504,0,10000991
-0,2012-01-09,FP-GTC,100847,0,13,99.8144,0,10000782
-0,2013-09-09,Auction,73506,0,16,36.6553,0,10000835
-0,2013-07-30,FP-non GTC,67703,3,16,3.5994,0,10000203
-0,2012-05-09,Others,139973,0,13,8.8491,0,10000074
-0,2012-10-13,FP-GTC,20865,0,14,19.0378,0,10000747
-0,2012-09-10,FP-GTC,158798,0,11,37.8799,0,10000312
-0,2012-07-25,FP-GTC,95173,0,14,83.3301,0,10000893
-0,2013-08-19,Auction,166013,15,16,41.9336,0,10000846
-0,2012-02-13,Others,87118,0,12,93.4500,0,10000136
-0,2013-07-06,FP-non GTC,1120,3,-99,45.6255,0,10000086
-0,2012-12-20,Auction,50677,0,12,15.1782,0,10000452
-0,2013-08-20,FP-non GTC,165888,0,11,63.8546,0,10000780
-0,2012-11-29,FP-GTC,63861,0,14,35.9757,0,10000134
-0,2013-09-10,Auction,95672,0,-99,13.9615,0,10000553
-0,2013-12-11,Others,314,0,14,61.2280,0,10000269
-0,2013-05-27,FP-GTC,1504,0,16,89.8299,0,10000168
-0,2012-06-22,FP-non GTC,94847,0,11,86.5431,0,10000641
-0,2012-02-24,FP-non GTC,314,0,14,7.0223,0,10000288
-0,2013-01-12,FP-GTC,20865,0,15,99.4581,0,10000069
-0,2013-10-28,Auction,174106,3,15,19.0917,0,10000198
-0,2012-01-29,Auction,121153,0,11,16.5034,0,10000492
-0,2012-12-02,FP-non GTC,53064,0,5,40.0250,0,10000002
-0,2013-04-17,ABIN,60340,0,13,96.5255,0,10000357
-0,2013-12-16,Auction,61323,0,15,72.7839,0,10000195
-0,2012-06-20,Auction,37831,0,5,8.3551,0,10000429
-0,2012-11-22,ABIN,31387,3,14,17.2673,0,10000854
-0,2012-11-22,FP-GTC,95173,0,-99,70.2540,0,10000619
-0,2013-09-30,FP-non GTC,57990,0,11,81.0956,0,10000431
-0,2013-08-09,ABIN,155226,0,-99,41.4198,0,10000204
-0,2012-09-03,Auction,88750,0,11,45.2273,0,10000947
-0,2012-01-01,FP-non GTC,156356,0,12,44.7485,0,10000439
-0,2012-04-21,ABIN,50677,0,16,80.4618,0,10000466
-0,2013-03-30,Others,20213,0,12,86.3777,0,10000781
-0,2012-05-18,FP-GTC,12688,0,14,11.7047,0,10000550
-0,2013-08-18,Auction,31519,0,13,57.6148,0,10000608
-0,2013-10-27,FP-non GTC,24760,0,5,34.9687,0,10000322
-0,2013-03-26,FP-GTC,1349,0,11,51.2759,0,10000800
-0,2012-08-31,FP-GTC,4943,0,14,43.3525,0,10000916
-0,2013-10-27,FP-GTC,99985,0,13,91.0379,0,10000091
-0,2012-07-10,Others,24760,0,16,39.2775,0,10000813
-0,2012-11-06,FP-non GTC,50508,0,16,22.9019,0,10000271
-0,2013-11-27,FP-non GTC,175750,0,16,36.9125,0,10000898
-0,2013-12-22,FP-GTC,80053,0,13,83.0584,0,10000443
-0,2012-12-16,Auction,6762,0,14,97.8880,0,10000723
-0,2012-08-08,Others,57990,0,5,65.6268,0,10000715
-0,2012-07-23,FP-GTC,48027,0,-99,98.0398,0,10000565
-0,2012-12-20,FP-GTC,106246,0,12,67.1574,0,10000293
-0,2013-11-01,ABIN,1349,0,5,10.9577,0,10000369
-0,2012-04-28,Auction,139973,3,11,25.6617,0,10000067
-0,2012-11-01,FP-non GTC,155226,0,14,65.7990,0,10000083
-0,2013-05-27,FP-GTC,48027,0,13,61.6990,0,10000100
-0,2013-08-07,FP-GTC,314,0,14,30.0501,0,10000344
-0,2013-03-13,FP-non GTC,175750,0,-99,82.5221,0,10000683
-0,2013-07-30,FP-GTC,31519,3,16,95.1425,0,10000606
-0,2013-12-18,Auction,50508,0,15,93.7747,0,10000283
-0,2012-11-28,FP-non GTC,53064,0,11,65.5664,0,10000903
-0,2013-10-04,Others,26249,0,14,66.2702,0,10000149
-0,2012-02-01,ABIN,15687,0,-99,55.4663,0,10000025
-0,2012-11-09,Auction,132939,0,5,0.4841,0,10000477
-0,2012-05-31,FP-non GTC,67698,0,13,5.6112,0,10000205
-0,2012-01-10,ABIN,170083,3,14,91.3684,0,10000968
-0,2013-10-02,FP-GTC,41940,0,14,85.7042,0,10000503
-0,2013-06-20,Others,106340,15,11,64.5684,0,10000573
-0,2013-03-30,FP-GTC,87118,0,12,11.5656,0,10000456
-0,2013-05-20,Others,37831,0,16,76.5619,0,10000829
-0,2012-01-30,Others,16145,0,16,0.1008,0,10000079
-0,2013-12-08,FP-GTC,46575,0,14,44.8329,0,10000439
-0,2013-01-14,ABIN,73506,0,-99,14.5712,0,10000568
-0,2012-04-19,Others,130,0,13,73.6383,0,10000907
-0,2012-11-09,Others,11848,0,15,6.9025,0,10000095
-0,2013-03-15,FP-GTC,175750,0,5,55.2025,0,10000359
-0,2012-02-18,FP-non GTC,2635,0,-99,81.9584,0,10000506
-0,2013-12-07,Auction,80135,0,12,85.1453,0,10000262
-0,2013-04-19,ABIN,174106,3,11,94.1872,0,10000252
-0,2012-09-04,FP-non GTC,6762,0,16,79.4180,0,10000819
-0,2012-11-21,Others,37831,0,15,72.2244,0,10000911
-0,2013-06-03,Auction,164261,0,16,3.6716,0,10000112
-0,2013-01-23,FP-non GTC,121153,0,11,85.4725,0,10000843
-0,2013-12-30,Auction,3838,0,5,28.5619,0,10000995
-0,2013-09-26,ABIN,95672,0,15,21.3948,0,10000679
-0,2013-06-07,FP-non GTC,10866,0,-99,92.6345,0,10000466
-0,2013-11-04,FP-non GTC,63861,0,13,60.7962,0,10000174
-0,2012-07-14,Others,48027,0,16,46.3174,0,10000603
-0,2013-04-28,ABIN,87118,0,11,8.2679,0,10000590
-0,2013-07-22,ABIN,41940,0,13,76.5107,0,10000159
-0,2012-03-03,Auction,44079,0,11,95.7853,0,10000806
-0,2013-09-20,Others,20213,0,5,35.7134,0,10000922
-0,2012-09-13,Auction,62179,0,-99,80.7493,0,10000356
-0,2012-06-15,ABIN,314,0,13,66.0961,0,10000139
-0,2013-01-29,Others,24541,0,-99,66.4687,0,10000803
-0,2013-06-19,Others,1357,0,15,35.6912,0,10000723
-0,2012-11-15,Others,80135,0,11,44.1951,0,10000083
-0,2012-10-22,Others,67698,0,12,6.8778,0,10000535
-0,2013-09-09,FP-non GTC,50508,0,16,31.1656,0,10000160
-0,2013-10-22,Others,3838,0,5,98.1652,0,10000875
-0,2012-10-11,Others,1349,0,15,31.2521,0,10000663
-0,2013-08-30,FP-GTC,20485,0,-99,15.8748,0,10000988
-0,2013-04-10,Others,38238,0,15,29.3240,0,10000686
-0,2013-07-20,Auction,24760,0,-99,57.0828,0,10000657
-0,2012-08-04,FP-GTC,159184,0,12,51.4276,0,10000895
-0,2013-06-28,FP-non GTC,13836,0,-99,27.7177,0,10000897
-0,2013-05-06,FP-non GTC,43398,0,5,51.7179,0,10000862
-0,2013-02-01,ABIN,15868,0,5,46.3238,0,10000744
-0,2013-05-12,Auction,533,0,5,83.6965,0,10000200
-0,2012-04-26,Auction,4943,0,16,59.3741,0,10000918
-0,2012-03-12,Others,48027,0,14,76.2845,0,10000564
-0,2012-08-15,ABIN,32876,0,-99,72.1322,0,10000483
-0,2012-03-03,Others,43479,0,15,40.9293,0,10000937
-0,2013-01-20,Auction,38238,0,15,59.0009,0,10000040
-0,2012-03-17,Others,38238,0,12,59.9178,0,10000267
-0,2012-03-30,ABIN,50508,0,11,27.4960,0,10000970
-0,2012-04-22,ABIN,53064,0,-99,48.6321,0,10000750
-0,2012-05-29,Others,1504,0,11,98.6906,0,10000143
-0,2013-07-02,FP-non GTC,80135,0,11,40.2555,0,10000353
-0,2012-01-14,FP-GTC,6762,0,13,7.0226,0,10000466
-0,2012-11-19,Others,130,0,12,99.7620,0,10000731
-0,2012-06-23,Others,6762,0,5,63.1128,0,10000103
-0,2013-06-16,FP-non GTC,16145,3,12,64.2030,0,10000530
-0,2013-12-30,Auction,99985,0,5,23.4290,0,10000972
-0,2013-06-09,Others,46575,0,13,13.4803,0,10000745
-0,2013-12-10,Auction,103324,15,16,99.7648,0,10000786
-0,2012-09-28,ABIN,24760,0,15,11.3547,0,10000673
-0,2012-11-29,Others,158798,0,-99,82.8002,0,10000586
-0,2012-04-16,Others,175750,0,12,16.9311,0,10000751
-0,2012-03-16,Auction,46575,0,5,3.2964,0,10000589
-0,2012-01-24,ABIN,150265,15,14,64.6865,0,10000434
-0,2012-09-26,Others,38238,0,12,73.3389,0,10000438
-0,2012-01-30,FP-GTC,15868,0,-99,5.6161,0,10000671
-0,2013-05-25,FP-non GTC,38238,0,-99,93.9862,0,10000669
-0,2012-11-15,FP-non GTC,6762,0,14,19.2198,0,10000276
-0,2012-08-17,FP-non GTC,64076,0,14,88.5536,0,10000347
-0,2013-06-02,ABIN,67698,2,11,24.6629,0,10000487
-0,2012-02-14,FP-non GTC,175750,0,11,28.6018,0,10000292
-0,2013-03-29,Auction,314,0,12,14.1619,0,10000757
-0,2012-02-24,FP-GTC,12688,0,5,48.2185,0,10000829
-0,2013-06-20,FP-non GTC,67698,0,5,68.2468,0,10000640
-0,2013-02-12,FP-GTC,57784,0,13,80.4582,0,10000435
-0,2013-06-20,Auction,26249,0,-99,60.6520,0,10000950
-0,2013-04-25,Auction,152801,0,15,76.0514,0,10000612
-0,2012-09-15,FP-non GTC,32876,0,16,56.6683,0,10000146
-0,2012-05-03,Others,1357,0,-99,1.3769,0,10000268
-0,2012-01-14,Auction,9426,3,11,71.7337,0,10000443
-0,2012-08-12,FP-non GTC,139973,0,5,80.5935,0,10000932
-0,2012-11-27,Others,175750,0,5,54.8804,0,10000716
-0,2013-08-28,FP-non GTC,16145,3,-99,70.7167,0,10000244
-0,2012-09-23,Auction,20213,0,16,4.5554,0,10000608
-0,2013-10-15,ABIN,1504,0,5,38.8845,0,10000803
-0,2013-04-30,FP-non GTC,6762,0,14,47.3754,0,10000538
-0,2012-01-06,ABIN,15868,0,5,33.4885,0,10000847
-0,2012-04-30,ABIN,57990,0,5,63.8902,0,10000821
-0,2012-09-27,ABIN,38238,0,16,43.0875,0,10000524
-0,2012-06-30,FP-non GTC,152801,0,12,29.8433,0,10000102
-0,2013-05-25,ABIN,40059,3,5,43.0317,0,10000899
-0,2013-08-11,FP-non GTC,10866,0,15,50.1175,0,10000211
-0,2013-05-28,Auction,150265,15,14,79.0973,0,10000675
-0,2012-10-01,FP-non GTC,48904,0,13,28.9811,0,10000301
-0,2013-04-14,FP-GTC,103324,15,13,45.0386,0,10000853
-0,2012-07-23,Auction,20485,0,13,21.4452,0,10000116
-0,2013-01-11,ABIN,80287,0,5,90.5218,0,10000503
-0,2012-10-07,FP-GTC,25147,0,16,98.8443,0,10000667
-0,2012-12-12,FP-non GTC,100847,0,16,27.8064,0,10000075
-0,2012-04-24,ABIN,165888,0,-99,41.8494,0,10000323
-0,2012-01-13,FP-GTC,80053,0,14,30.7342,0,10000781
-0,2013-02-23,FP-GTC,121153,0,-99,50.0159,0,10000361
-0,2012-10-22,ABIN,1349,0,11,39.3281,0,10000951
-0,2012-01-20,ABIN,2635,0,-99,34.4591,0,10000433
-0,2012-04-14,FP-GTC,26262,0,13,34.2101,0,10000000
-0,2012-07-20,Others,12688,0,5,83.1926,0,10000755
-0,2012-09-12,FP-non GTC,216,0,11,61.5914,0,10000504
-0,2012-01-29,Others,13987,0,5,49.9596,0,10000340
-0,2013-01-10,ABIN,65,0,12,84.5331,0,10000988
-0,2012-08-20,FP-GTC,2023,0,16,46.2586,0,10000802
-0,2013-11-24,Others,156614,0,-99,68.9025,0,10000372
-0,2012-12-02,Auction,158798,0,16,62.5343,0,10000020
-0,2012-06-12,FP-non GTC,175750,0,14,21.6343,0,10000451
-0,2013-02-11,ABIN,9426,3,14,54.9943,0,10000615
-0,2012-11-01,FP-non GTC,88750,0,-99,16.7597,0,10000145
-0,2013-08-16,Auction,15868,0,15,14.8127,0,10000190
-0,2013-07-30,Auction,45333,0,14,67.5141,0,10000870
-0,2012-01-05,Others,24541,0,15,92.5069,0,10000821
-0,2013-04-30,Auction,57990,3,16,39.4144,0,10000140
-0,2013-01-03,Auction,314,0,14,73.2865,0,10000848
-0,2012-02-05,ABIN,38238,0,5,21.4669,0,10000158
-0,2012-05-08,FP-non GTC,13987,0,16,15.7925,0,10000816
-0,2012-12-08,Others,164261,0,12,33.2040,0,10000614
-0,2013-08-14,FP-non GTC,139255,15,16,37.1862,0,10000745
-0,2012-08-29,Auction,156614,0,14,11.0487,0,10000181
-0,2013-09-20,FP-GTC,95173,0,-99,42.2822,0,10000642
-0,2013-01-02,Auction,43479,0,5,21.7236,0,10000900
-0,2012-11-18,FP-GTC,139973,0,11,9.9099,0,10000191
-0,2012-02-03,Others,40059,3,5,74.0231,0,10000422
-0,2012-04-28,FP-non GTC,3838,0,5,6.5803,0,10000162
-0,2013-05-23,Others,80287,0,14,79.3278,0,10000474
-0,2012-06-16,Auction,108782,15,13,15.8824,0,10000259
-0,2013-12-05,FP-non GTC,63861,0,-99,78.2662,0,10000510
-0,2013-02-14,ABIN,31519,0,5,98.5361,0,10000591
-0,2012-07-27,ABIN,38238,0,16,97.9582,0,10000396
-0,2012-09-20,FP-non GTC,31519,0,13,15.0370,0,10000080
-0,2013-09-16,Auction,45333,0,15,99.9888,0,10000082
-0,2013-03-23,ABIN,1357,0,-99,78.1418,0,10000091
-0,2013-04-11,FP-GTC,45333,0,11,58.2670,0,10000078
-0,2012-01-23,Auction,63864,3,12,78.3096,0,10000755
-0,2012-01-27,FP-non GTC,15115,0,13,62.3493,0,10000191
-0,2012-10-30,FP-non GTC,1357,0,16,75.3922,0,10000178
-0,2012-05-22,Auction,75665,0,16,25.2559,0,10000662
-0,2013-08-22,Auction,51582,0,11,22.2512,0,10000937
-0,2012-10-20,ABIN,139973,0,-99,5.9459,0,10000542
-0,2013-04-06,FP-GTC,15868,0,5,44.5247,0,10000090
-0,2012-09-06,Others,43479,0,13,22.0043,0,10000822
-0,2012-05-19,ABIN,31519,0,12,67.8089,0,10000046
-0,2012-08-13,Auction,57990,3,12,51.2947,0,10000472
-0,2012-09-18,FP-GTC,139973,0,15,50.6074,0,10000482
-0,2013-02-27,Auction,13836,0,5,12.9600,0,10000057
-0,2012-10-02,Others,53064,0,11,21.0200,0,10000142
-0,2013-02-23,FP-GTC,26262,0,11,18.1814,0,10000457
-0,2012-10-05,FP-non GTC,100847,0,16,17.3068,0,10000061
-0,2013-05-03,FP-GTC,99985,0,11,88.9624,0,10000299
-0,2012-05-22,Others,11848,0,13,25.6477,0,10000572
-0,2013-01-21,ABIN,33038,15,16,37.8162,0,10000036
-0,2013-09-22,FP-non GTC,57990,0,12,72.4322,0,10000304
-0,2013-09-22,ABIN,65,0,13,5.8659,0,10000424
-0,2012-08-09,ABIN,44079,0,5,65.9028,0,10000790
-0,2013-08-05,Auction,1504,0,13,87.7795,0,10000621
-0,2013-08-06,FP-non GTC,88750,0,5,47.7266,0,10000579
-0,2013-11-22,Auction,62179,0,16,17.7918,0,10000626
-0,2013-08-19,ABIN,16145,0,16,56.5899,0,10000818
-0,2012-09-24,ABIN,4943,0,16,85.8548,0,10000515
-0,2013-05-14,ABIN,43479,0,5,12.9686,0,10000527
-0,2013-07-15,Others,73506,0,-99,88.5967,0,10000266
-0,2012-11-25,FP-GTC,64076,0,14,28.2347,0,10000361
-0,2012-01-07,ABIN,80053,0,13,77.0661,0,10000149
-0,2013-06-08,FP-non GTC,80287,0,-99,98.5548,0,10000668
-0,2013-10-03,ABIN,88750,0,11,34.1582,0,10000116
-0,2012-04-26,Auction,175750,0,12,53.9184,0,10000084
-0,2013-01-31,FP-non GTC,15868,0,12,71.5902,0,10000372
-0,2013-04-23,FP-non GTC,32876,0,14,94.0781,0,10000798
-0,2013-10-20,ABIN,38238,0,13,43.3655,0,10000837
-0,2013-04-21,ABIN,1504,0,5,32.5360,0,10000954
-0,2013-03-05,FP-non GTC,156356,0,11,48.8941,0,10000480
-0,2013-05-08,FP-non GTC,139973,0,16,94.5155,0,10000278
-0,2013-09-06,FP-GTC,45333,0,
<TRUNCATED>
[31/50] [abbrv] kylin git commit: KYLIN-2084 Unload sample table
failed
Posted by li...@apache.org.
KYLIN-2084 Unload sample table failed
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef44b7f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef44b7f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef44b7f6
Branch: refs/heads/master-cdh5.7
Commit: ef44b7f6873b2e5ac55243818877474cdc0376d8
Parents: c3bfd77
Author: Jason <ji...@163.com>
Authored: Mon Oct 17 13:18:57 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Mon Oct 17 13:20:16 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/metadata/MetadataManager.java | 10 ++++++++++
.../org/apache/kylin/rest/controller/TableController.java | 3 +++
.../java/org/apache/kylin/rest/service/ModelService.java | 7 +++++++
3 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 92d41b9..e0c78ce 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -496,6 +496,16 @@ public class MetadataManager {
return false;
}
+ public List<String> getModelsUsingTable(String tableName, String projectName) throws IOException {
+ List<String> models = new ArrayList<>();
+ for (DataModelDesc modelDesc : getModels(projectName)) {
+ if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
+ models.add(modelDesc.getName());
+ }
+ }
+ return models;
+ }
+
public boolean isTableInAnyModel(String tableName) {
for (DataModelDesc modelDesc : getModels()) {
if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 34cc57f..d9050c1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -197,6 +197,9 @@ public class TableController extends BasicController {
if (!modelService.isTableInModel(tableName, project)) {
cubeMgmtService.removeTableFromProject(tableName, project);
rtn = true;
+ }else{
+ List<String> models = modelService.getModelsUsingTable(tableName, project);
+ throw new InternalErrorException("Table is already in use by models "+models);
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index bfff619..2f8667c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -132,4 +132,11 @@ public class ModelService extends BasicService {
tableName = dbTableName[0] + "." + dbTableName[1];
return getMetadataManager().isTableInModel(tableName, projectName);
}
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#desc, 'ADMINISTRATION') or hasPermission(#desc, 'MANAGEMENT')")
+ public List<String> getModelsUsingTable(String tableName, String projectName) throws IOException {
+ String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
+ tableName = dbTableName[0] + "." + dbTableName[1];
+ return getMetadataManager().getModelsUsingTable(tableName, projectName);
+ }
}
[23/50] [abbrv] kylin git commit: Revert "KYLIN-2012 rename
RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN"
Posted by li...@apache.org.
Revert "KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN"
This reverts commit 5974fc0870be17a8801c55c7496093d42dbb7c4f.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bebbc784
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bebbc784
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bebbc784
Branch: refs/heads/master-cdh5.7
Commit: bebbc78476c6b41b50fc0492134a5418d522533e
Parents: e6698c8
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 14 11:46:01 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 14 11:46:01 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/CubeDescManager.java | 4 ++--
.../src/main/java/org/apache/kylin/cube/CubeInstance.java | 2 +-
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 6 +++---
.../java/org/apache/kylin/cube/CubeManagerCacheTest.java | 4 ++--
.../kylin/metadata/realization/RealizationStatusEnum.java | 2 +-
.../org/apache/kylin/rest/controller/CubeController.java | 2 +-
.../java/org/apache/kylin/rest/service/JobService.java | 2 +-
webapp/app/partials/cubes/cubes.html | 10 +++++-----
8 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 602707f..d6364fe 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -158,10 +158,10 @@ public class CubeDescManager {
cubeDescMap.putLocal(ndesc.getName(), ndesc);
Cuboid.reloadCache(name);
- // if related cube is in BROKEN state before, change it back to DISABLED
+ // if related cube is in DESCBROKEN state before, change it back to DISABLED
CubeManager cubeManager = CubeManager.getInstance(config);
for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
- if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+ if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
cubeManager.reloadCubeLocal(cube.getName());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index b8e9394..720690d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -153,7 +153,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
// in a temporary broken state, so that user can edit and fix it. Broken state is often due to
// schema changes at source.
public boolean allowBrokenDescriptor() {
- return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.BROKEN) && segments.isEmpty();
+ return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
}
public String getResourcePath() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 1ebeb60..c558c6d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -994,12 +994,12 @@ public class CubeManager implements IRealizationProvider {
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
if (!cubeDesc.getError().isEmpty()) {
- cube.setStatus(RealizationStatusEnum.BROKEN);
+ cube.setStatus(RealizationStatusEnum.DESCBROKEN);
logger.warn("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), cubeName);
- } else if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+ } else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
cube.setStatus(RealizationStatusEnum.DISABLED);
- logger.info("cube {} changed from BROKEN to DISABLED", cubeName);
+ logger.info("cube {} changed from DESCBROKEN to DISABLED", cubeName);
}
cube.setConfig((KylinConfigExt) cubeDesc.getConfig());
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index 3a2188f..3f3b462 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -65,11 +65,11 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube");
assertEquals(0, createdCube.getSegments().size());
assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
- createdCube.setStatus(RealizationStatusEnum.BROKEN);
+ createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
CubeUpdate cubeBuilder = new CubeUpdate(createdCube);
cubeManager.updateCube(cubeBuilder);
- assertEquals(RealizationStatusEnum.BROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
+ assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
public CubeDescManager getCubeDescManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
index 2f3d5ca..27e2d57 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
@@ -20,6 +20,6 @@ package org.apache.kylin.metadata.realization;
public enum RealizationStatusEnum {
- DISABLED, READY, BROKEN
+ DISABLED, READY, DESCBROKEN
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index cae6e0f..be242c3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -339,7 +339,7 @@ public class CubeController extends BasicController {
if (cube == null) {
throw new BadRequestException("Cannot find cube " + cubeName);
}
- if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+ if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
throw new BadRequestException("Broken cube can't be cloned");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index d32ff00..589f67d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -201,7 +201,7 @@ public class JobService extends BasicService {
public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException, JobException {
- if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+ if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
throw new BadRequestException("Broken cube " + cube.getName() + " can't be built");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index a667058..de9b99b 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -67,7 +67,7 @@
</td>
<td>
<span class="label"
- ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED', 'label-warning': cube.status=='BROKEN'}">
+ ng-class="{'label-success': cube.status=='READY', 'label-default': cube.status=='DISABLED', 'label-warning': cube.status=='DESCBROKEN'}">
{{ cube.status}}
</span>
</td>
@@ -94,13 +94,13 @@
<li ng-if="cube.status!='READY' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))">
<a ng-click="cubeEdit(cube);">Edit</a></li>
<li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li>
- <li ng-if="cube.status!='BROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li>
- <li ng-if="cube.status!='BROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li>
- <li ng-if="cube.status!='BROKEN'"><a ng-click="startMerge(cube)">Merge</a></li>
+ <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li>
+ <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li>
+ <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startMerge(cube)">Merge</a></li>
<li ng-if="cube.status=='READY'"><a ng-click="disable(cube)">Disable</a></li>
<li ng-if="cube.status=='DISABLED'"><a ng-click="enable(cube)">Enable</a></li>
<li ng-if="cube.status=='DISABLED'"><a ng-click="purge(cube)">Purge</a></li>
- <li ng-if="cube.status!='BROKEN'"><a ng-click="cloneCube(cube)">Clone</a></li>
+ <li ng-if="cube.status!='DESCBROKEN'"><a ng-click="cloneCube(cube)">Clone</a></li>
</ul>
</div>
[26/50] [abbrv] kylin git commit: KYLIN-2088 Support intersect count
for calculation of retention or conversion rates
Posted by li...@apache.org.
KYLIN-2088 Support intersect count for calculation of retention or conversion rates
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4c970ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4c970ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4c970ad
Branch: refs/heads/master-cdh5.7
Commit: b4c970adf18362daade77e936693dac08c0639e1
Parents: 61a08d4
Author: sunyerui <su...@gmail.com>
Authored: Wed Oct 12 20:59:54 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Oct 16 08:10:05 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +-
.../org/apache/kylin/measure/MeasureType.java | 5 ++
.../kylin/measure/bitmap/BitmapCounter.java | 32 +++++++
.../BitmapIntersectDistinctCountAggFunc.java | 94 ++++++++++++++++++++
.../kylin/measure/bitmap/BitmapMeasureType.java | 9 ++
.../apache/kylin/query/ITKylinQueryTest.java | 6 ++
.../query/sql_intersect_count/query00.sql | 32 +++++++
.../kylin/query/relnode/OLAPAggregateRel.java | 16 +++-
8 files changed, 195 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 99c3c5a..7dacd06 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -765,7 +765,9 @@ abstract public class KylinConfigBase implements Serializable {
}
public Map<String, String> getUDFs() {
- return getPropertiesByPrefix("kylin.query.udf.");
+ Map<String, String> udfMap = getPropertiesByPrefix("kylin.query.udf.");
+ udfMap.put("intersect_count", "org.apache.kylin.measure.bitmap.BitmapIntersectDistinctCountAggFunc");
+ return udfMap;
}
public int getHBaseMaxConnectionThreads() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 82618e9..e7312f2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -115,6 +115,11 @@ abstract public class MeasureType<T> {
/** Returns a Calcite aggregation function implementation class */
abstract public Class<?> getRewriteCalciteAggrFunctionClass();
+ /** Some measure may return different class depends on call name, eg. BitmapMeasureType */
+ public Class<?> getRewriteCalciteAggrFunctionClass(String callName) {
+ return getRewriteCalciteAggrFunctionClass();
+ }
+
/* ============================================================================
* Storage
* ---------------------------------------------------------------------------- */
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index d3b57a7..827390d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -47,6 +47,12 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
bitmap.clear();
}
+ public BitmapCounter clone() {
+ BitmapCounter newCounter = new BitmapCounter();
+ newCounter.bitmap = bitmap.clone();
+ return newCounter;
+ }
+
public void add(int value) {
bitmap.add(value);
}
@@ -74,6 +80,10 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
this.bitmap.or(another.bitmap);
}
+ public void intersect(BitmapCounter another) {
+ this.bitmap.and(another.bitmap);
+ }
+
public long getCount() {
return this.bitmap.getCardinality();
}
@@ -107,6 +117,28 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
@Override
+ public String toString() {
+ long count = getCount();
+ if (count <= 10) {
+ return "(" + count + ")" + bitmap.toString();
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append("(").append(count).append("){");
+ int values = 0;
+ for (Integer v : bitmap) {
+ if (values++ < 10) {
+ sb.append(v).append(",");
+ } else {
+ sb.append("...");
+ break;
+ }
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+ }
+
+ @Override
public int hashCode() {
final int prime = 31;
int result = 1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
new file mode 100644
index 0000000..cf42d1b
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.bitmap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * BitmapIntersectDistinctCountAggFunc is an UDAF used for calculating the intersection of two or more bitmaps
+ * Usage: intersect_count(columnToCount, columnToFilter, filterList)
+ * Example: intersect_count(uuid, event, array['A', 'B', 'C']), meaning find the count of uuid in all A/B/C 3 bitmaps
+ * requires an bitmap count distinct measure of uuid, and an dimension of event
+ */
+public class BitmapIntersectDistinctCountAggFunc {
+ private static final Logger logger = LoggerFactory.getLogger(BitmapIntersectDistinctCountAggFunc.class);
+
+ public static class RetentionPartialResult {
+ Map<Object, BitmapCounter> map;
+ List keyList;
+
+ public RetentionPartialResult() {
+ map = new LinkedHashMap<>();
+ }
+
+ public void add(Object key, List keyList, Object value) {
+ if (this.keyList == null) {
+ this.keyList = keyList;
+ }
+ BitmapCounter counter = map.get(key);
+ if (counter == null) {
+ counter = new BitmapCounter();
+ map.put(key, counter);
+ }
+ counter.merge((BitmapCounter)value);
+ }
+
+ public long result() {
+ if (keyList == null || keyList.isEmpty()) {
+ return 0;
+ }
+ BitmapCounter counter = null;
+ for (Object key : keyList) {
+ BitmapCounter c = map.get(key);
+ if (c == null) {
+ // We have a key in filter list but not in map, meaning there's no intersect data
+ return 0;
+ } else {
+ if (counter == null) {
+ counter = c.clone();
+ }
+ counter.intersect(c);
+ }
+ }
+ return counter.getCount();
+ }
+ }
+
+ public static RetentionPartialResult init() {
+ return new RetentionPartialResult();
+ }
+
+ public static RetentionPartialResult add(RetentionPartialResult result, Object value, Object key, List keyList) {
+ result.add(key, keyList, value);
+ return result;
+ }
+
+ public static RetentionPartialResult merge(RetentionPartialResult result, Object value, Object key, List keyList) {
+ return add(result, value, key, keyList);
+ }
+
+ public static long result(RetentionPartialResult result) {
+ return result.result();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index be96eb5..2b88e21 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -39,6 +39,7 @@ import org.apache.kylin.metadata.model.TblColRef;
*/
public class BitmapMeasureType extends MeasureType<BitmapCounter> {
public static final String FUNC_COUNT_DISTINCT = "COUNT_DISTINCT";
+ public static final String FUNC_INTERSECT_COUNT_DISTINCT = "INTERSECT_COUNT";
public static final String DATATYPE_BITMAP = "bitmap";
public static class Factory extends MeasureTypeFactory<BitmapCounter> {
@@ -160,6 +161,14 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
return BitmapDistinctCountAggFunc.class;
}
+ @Override
+ public Class<?> getRewriteCalciteAggrFunctionClass(String callName) {
+ if (callName != null && callName.equalsIgnoreCase(FUNC_INTERSECT_COUNT_DISTINCT)) {
+ return BitmapIntersectDistinctCountAggFunc.class;
+ }
+ return BitmapDistinctCountAggFunc.class;
+ }
+
// In order to keep compatibility with old version, tinyint/smallint/int column use value directly, without dictionary
private boolean needDictionaryColumn(FunctionDesc functionDesc) {
DataType dataType = functionDesc.getParameter().getColRefs().get(0).getType();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 59a3a04..a0706ca 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -355,4 +355,10 @@ public class ITKylinQueryTest extends KylinTestBase {
this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_window");
}
+ @Test
+ public void testIntersectCountQuery() throws Exception {
+ // cannot compare coz H2 does not support intersect count yet..
+ this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_intersect_count");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql b/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
new file mode 100644
index 0000000..15e274a
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
@@ -0,0 +1,32 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select
+week_beg_dt as week,
+intersect_count( seller_id, lstg_format_name, array['FP-GTC']) as a,
+intersect_count( seller_id, lstg_format_name, array['Auction']) as b,
+intersect_count( seller_id, lstg_format_name, array['Others']) as c,
+intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Auction']) as ab,
+intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Others']) as ac,
+intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Auction', 'Others']) as abc,
+count(distinct seller_id) as sellers,
+count(*) as cnt
+from test_kylin_fact left join edw.test_cal_dt on test_kylin_fact.cal_dt = edw.test_cal_dt.CAL_DT
+where week_beg_dt in (DATE '2013-12-22', DATE '2012-06-23')
+group by week_beg_dt
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/b4c970ad/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 97efb27..8ecb808 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -56,6 +56,7 @@ import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.measure.bitmap.BitmapMeasureType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -79,6 +80,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
AGGR_FUNC_MAP.put("$SUM0", "SUM");
AGGR_FUNC_MAP.put("COUNT", "COUNT");
AGGR_FUNC_MAP.put("COUNT_DISTINCT", "COUNT_DISTINCT");
+ AGGR_FUNC_MAP.put(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT, "COUNT_DISTINCT");
AGGR_FUNC_MAP.put("MAX", "MAX");
AGGR_FUNC_MAP.put("MIN", "MIN");
@@ -224,6 +226,15 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
Set<TblColRef> columns = inputColumnRowType.getSourceColumnsByIndex(i);
this.groups.addAll(columns);
}
+ // Some UDAF may group data by itself, add group key into groups, prevents aggregate at cube storage server side
+ for (AggregateCall aggCall : this.rewriteAggCalls) {
+ String aggregateName = aggCall.getAggregation().getName();
+ if (aggregateName.equalsIgnoreCase(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)) {
+ int index = aggCall.getArgList().get(1);
+ TblColRef column = inputColumnRowType.getColumnByIndex(index);
+ groups.add(column);
+ }
+ }
}
private void buildAggregations() {
@@ -380,16 +391,17 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
// rebuild function
+ String callName = aggCall.getAggregation().getName();
RelDataType fieldType = aggCall.getType();
SqlAggFunction newAgg = aggCall.getAggregation();
if (func.isCount()) {
newAgg = SqlStdOperatorTable.SUM0;
} else if (func.getMeasureType().getRewriteCalciteAggrFunctionClass() != null) {
- newAgg = createCustomAggFunction(func.getExpression(), fieldType, func.getMeasureType().getRewriteCalciteAggrFunctionClass());
+ newAgg = createCustomAggFunction(callName, fieldType, func.getMeasureType().getRewriteCalciteAggrFunctionClass(callName));
}
// rebuild aggregate call
- AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, newAgg.getName());
+ AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, callName);
return newAggCall;
}
[03/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming
code
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
deleted file mode 100644
index 271bf41..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.cachesync.Broadcaster;
-import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
-import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class StreamingManager {
-
- private static final Logger logger = LoggerFactory.getLogger(StreamingManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, StreamingManager> CACHE = new ConcurrentHashMap<KylinConfig, StreamingManager>();
-
- public static final Serializer<StreamingConfig> STREAMING_SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class);
-
- private KylinConfig config;
-
- // name ==> StreamingConfig
- private CaseInsensitiveStringCache<StreamingConfig> streamingMap;
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- private StreamingManager(KylinConfig config) throws IOException {
- this.config = config;
- this.streamingMap = new CaseInsensitiveStringCache<StreamingConfig>(config, "streaming");
-
- // touch lower level metadata before registering my listener
- reloadAllStreaming();
- Broadcaster.getInstance(config).registerListener(new StreamingSyncListener(), "streaming");
- }
-
- private class StreamingSyncListener extends Broadcaster.Listener {
- @Override
- public void onClearAll(Broadcaster broadcaster) throws IOException {
- clearCache();
- }
-
- @Override
- public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException {
- if (event == Event.DROP)
- removeStreamingLocal(cacheKey);
- else
- reloadStreamingConfigLocal(cacheKey);
- }
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- public static StreamingManager getInstance(KylinConfig config) {
- StreamingManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (StreamingManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new StreamingManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init StreamingManager from " + config, e);
- }
- }
- }
-
- private static String formatStreamingConfigPath(String name) {
- return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
- }
-
- private static String formatStreamingOutputPath(String streaming, int partition) {
- return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + partition + ".json";
- }
-
- private static String formatStreamingOutputPath(String streaming, List<Integer> partitions) {
- return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + StringUtils.join(partitions, "_") + ".json";
- }
-
- public StreamingConfig getStreamingConfig(String name) {
- return streamingMap.get(name);
- }
-
- public List<StreamingConfig> listAllStreaming() {
- return new ArrayList<>(streamingMap.values());
- }
-
- /**
- * Reload StreamingConfig from resource store It will be triggered by an desc
- * update event.
- *
- * @param name
- * @throws IOException
- */
- public StreamingConfig reloadStreamingConfigLocal(String name) throws IOException {
-
- // Save Source
- String path = StreamingConfig.concatResourcePath(name);
-
- // Reload the StreamingConfig
- StreamingConfig ndesc = loadStreamingConfigAt(path);
-
- // Here replace the old one
- streamingMap.putLocal(ndesc.getName(), ndesc);
- return ndesc;
- }
-
- // remove streamingConfig
- public void removeStreamingConfig(StreamingConfig streamingConfig) throws IOException {
- String path = streamingConfig.getResourcePath();
- getStore().deleteResource(path);
- streamingMap.remove(streamingConfig.getName());
- }
-
- public StreamingConfig getConfig(String name) {
- name = name.toUpperCase();
- return streamingMap.get(name);
- }
-
- public void removeStreamingLocal(String streamingName) {
- streamingMap.removeLocal(streamingName);
- }
-
- /**
- * Update CubeDesc with the input. Broadcast the event into cluster
- *
- * @param desc
- * @return
- * @throws IOException
- */
- public StreamingConfig updateStreamingConfig(StreamingConfig desc) throws IOException {
- // Validate CubeDesc
- if (desc.getUuid() == null || desc.getName() == null) {
- throw new IllegalArgumentException("SteamingConfig Illegal.");
- }
- String name = desc.getName();
- if (!streamingMap.containsKey(name)) {
- throw new IllegalArgumentException("StreamingConfig '" + name + "' does not exist.");
- }
-
- // Save Source
- String path = desc.getResourcePath();
- getStore().putResource(path, desc, STREAMING_SERIALIZER);
-
- // Reload the StreamingConfig
- StreamingConfig ndesc = loadStreamingConfigAt(path);
- // Here replace the old one
- streamingMap.put(ndesc.getName(), desc);
-
- return ndesc;
- }
-
- public StreamingConfig saveStreamingConfig(StreamingConfig streamingConfig) throws IOException {
- if (streamingConfig == null || StringUtils.isEmpty(streamingConfig.getName())) {
- throw new IllegalArgumentException();
- }
-
- if (streamingMap.containsKey(streamingConfig.getName()))
- throw new IllegalArgumentException("StreamingConfig '" + streamingConfig.getName() + "' already exists");
-
- String path = StreamingConfig.concatResourcePath(streamingConfig.getName());
- getStore().putResource(path, streamingConfig, StreamingConfig.SERIALIZER);
- streamingMap.put(streamingConfig.getName(), streamingConfig);
- return streamingConfig;
- }
-
- private StreamingConfig loadStreamingConfigAt(String path) throws IOException {
- ResourceStore store = getStore();
- StreamingConfig streamingDesc = store.getResource(path, StreamingConfig.class, STREAMING_SERIALIZER);
-
- if (StringUtils.isBlank(streamingDesc.getName())) {
- throw new IllegalStateException("StreamingConfig name must not be blank");
- }
- return streamingDesc;
- }
-
- private void reloadAllStreaming() throws IOException {
- ResourceStore store = getStore();
- logger.info("Reloading Streaming Metadata from folder " + store.getReadableResourcePath(ResourceStore.STREAMING_RESOURCE_ROOT));
-
- streamingMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.STREAMING_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- StreamingConfig streamingConfig;
- try {
- streamingConfig = loadStreamingConfigAt(path);
- } catch (Exception e) {
- logger.error("Error loading streaming desc " + path, e);
- continue;
- }
- if (path.equals(streamingConfig.getResourcePath()) == false) {
- logger.error("Skip suspicious desc at " + path + ", " + streamingConfig + " should be at " + streamingConfig.getResourcePath());
- continue;
- }
- if (streamingMap.containsKey(streamingConfig.getName())) {
- logger.error("Dup StreamingConfig name '" + streamingConfig.getName() + "' on path " + path);
- continue;
- }
-
- streamingMap.putLocal(streamingConfig.getName(), streamingConfig);
- }
-
- logger.debug("Loaded " + streamingMap.size() + " StreamingConfig(s)");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
deleted file mode 100644
index 32030ad..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.streaming.cli;
-
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.engine.streaming.monitor.StreamingMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class MonitorCLI {
-
- private static final Logger logger = LoggerFactory.getLogger(MonitorCLI.class);
-
- public static void main(String[] args) {
- Preconditions.checkArgument(args[0].equals("monitor"));
-
- int i = 1;
- List<String> receivers = null;
- String host = null;
- String tableName = null;
- String authorization = null;
- String cubeName = null;
- String projectName = "default";
- while (i < args.length) {
- String argName = args[i];
- switch (argName) {
- case "-receivers":
- receivers = Lists.newArrayList(StringUtils.split(args[++i], ";"));
- break;
- case "-host":
- host = args[++i];
- break;
- case "-tableName":
- tableName = args[++i];
- break;
- case "-authorization":
- authorization = args[++i];
- break;
- case "-cubeName":
- cubeName = args[++i];
- break;
- case "-projectName":
- projectName = args[++i];
- break;
- default:
- throw new RuntimeException("invalid argName:" + argName);
- }
- i++;
- }
- Preconditions.checkArgument(receivers != null && receivers.size() > 0);
- final StreamingMonitor streamingMonitor = new StreamingMonitor();
- if (tableName != null) {
- logger.info(String.format("check query tableName:%s host:%s receivers:%s", tableName, host, StringUtils.join(receivers, ";")));
- Preconditions.checkNotNull(host);
- Preconditions.checkNotNull(authorization);
- Preconditions.checkNotNull(tableName);
- streamingMonitor.checkCountAll(receivers, host, authorization, projectName, tableName);
- }
- if (cubeName != null) {
- logger.info(String.format("check cube cubeName:%s receivers:%s", cubeName, StringUtils.join(receivers, ";")));
- streamingMonitor.checkCube(receivers, cubeName, host);
- }
- System.exit(0);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
deleted file mode 100644
index 1d66b41..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming.cli;
-
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.engine.streaming.BootstrapConfig;
-import org.apache.kylin.engine.streaming.OneOffStreamingBuilder;
-import org.apache.kylin.engine.streaming.monitor.StreamingMonitor;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class StreamingCLI {
-
- private static final Logger logger = LoggerFactory.getLogger(StreamingCLI.class);
-
- public static void main(String[] args) {
- try {
- Preconditions.checkArgument(args[0].equals("streaming"));
- Preconditions.checkArgument(args[1].equals("start"));
-
- int i = 2;
- BootstrapConfig bootstrapConfig = new BootstrapConfig();
- while (i < args.length) {
- String argName = args[i];
- switch (argName) {
- case "-start":
- bootstrapConfig.setStart(Long.parseLong(args[++i]));
- break;
- case "-end":
- bootstrapConfig.setEnd(Long.parseLong(args[++i]));
- break;
- case "-cube":
- bootstrapConfig.setCubeName(args[++i]);
- break;
- case "-fillGap":
- bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
- break;
- case "-maxFillGapRange":
- bootstrapConfig.setMaxFillGapRange(Long.parseLong(args[++i]));
- break;
- default:
- logger.warn("ignore this arg:" + argName);
- }
- i++;
- }
- if (bootstrapConfig.isFillGap()) {
- final List<Pair<Long, Long>> gaps = StreamingMonitor.findGaps(bootstrapConfig.getCubeName());
- logger.info("all gaps:" + StringUtils.join(gaps, ","));
- for (Pair<Long, Long> gap : gaps) {
- List<Pair<Long, Long>> splitGaps = splitGap(gap, bootstrapConfig.getMaxFillGapRange());
- for (Pair<Long, Long> splitGap : splitGaps) {
- logger.info("start filling the gap from " + splitGap.getFirst() + " to " + splitGap.getSecond());
- startOneOffCubeStreaming(bootstrapConfig.getCubeName(), splitGap.getFirst(), splitGap.getSecond());
- logger.info("finish filling the gap from " + splitGap.getFirst() + " to " + splitGap.getSecond());
- }
- }
- } else {
- startOneOffCubeStreaming(bootstrapConfig.getCubeName(), bootstrapConfig.getStart(), bootstrapConfig.getEnd());
- logger.info("streaming process finished, exit with 0");
- System.exit(0);
- }
- } catch (Exception e) {
- printArgsError(args);
- logger.error("error start streaming", e);
- System.exit(-1);
- }
- }
-
- private static List<Pair<Long, Long>> splitGap(Pair<Long, Long> gap, long maxFillGapRange) {
- List<Pair<Long, Long>> gaps = Lists.newArrayList();
- Long startTime = gap.getFirst();
-
- while (startTime < gap.getSecond()) {
- Long endTime = gap.getSecond() <= startTime + maxFillGapRange ? gap.getSecond() : startTime + maxFillGapRange;
- gaps.add(Pair.newPair(startTime, endTime));
- startTime = endTime;
- }
-
- return gaps;
- }
-
- private static void startOneOffCubeStreaming(String cubeName, long start, long end) {
- final Runnable runnable = new OneOffStreamingBuilder(RealizationType.CUBE, cubeName, start, end).build();
- runnable.run();
- }
-
- private static void printArgsError(String[] args) {
- logger.warn("invalid args:" + StringUtils.join(args, " "));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
deleted file mode 100644
index 350a5f8..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming.cube;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import javax.annotation.Nullable;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
-import org.apache.kylin.cube.inmemcubing.InMemCubeBuilder;
-import org.apache.kylin.cube.util.CubingUtils;
-import org.apache.kylin.engine.EngineFactory;
-import org.apache.kylin.engine.streaming.StreamingBatchBuilder;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.IBuildable;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class StreamingCubeBuilder implements StreamingBatchBuilder {
-
- private static final Logger logger = LoggerFactory.getLogger(StreamingCubeBuilder.class);
-
- private final String cubeName;
- private int processedRowCount = 0;
-
- public StreamingCubeBuilder(String cubeName) {
- this.cubeName = cubeName;
- }
-
- @Override
- public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter) {
- try {
- CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
- final IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeInstance.getDescriptor());
-
- LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue<List<String>>();
- InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(), flatDesc, dictionaryMap);
- final Future<?> future = Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, cuboidWriter));
- processedRowCount = streamingBatch.getMessages().size();
- for (StreamingMessage streamingMessage : streamingBatch.getMessages()) {
- blockingQueue.put(streamingMessage.getData());
- }
- blockingQueue.put(Collections.<String> emptyList());
- future.get();
- cuboidWriter.flush();
-
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- } catch (ExecutionException e) {
- throw new RuntimeException("error build cube from StreamingBatch", e.getCause());
- } catch (IOException e) {
- throw new RuntimeException("error build cube from StreamingBatch", e.getCause());
- } finally {
- try {
- cuboidWriter.close();
- } catch (IOException e) {
- throw new RuntimeException("error build cube from StreamingBatch", e.getCause());
- }
- }
- }
-
- @Override
- public IBuildable createBuildable(StreamingBatch streamingBatch) {
- CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
- try {
- CubeSegment segment = cubeManager.appendSegment(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond());
- segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
- segment.setInputRecords(streamingBatch.getMessages().size());
- segment.setLastBuildTime(System.currentTimeMillis());
- return segment;
- } catch (IOException e) {
- throw new RuntimeException("failed to create IBuildable", e);
- }
- }
-
- @Override
- public Map<Long, HyperLogLogPlusCounter> sampling(StreamingBatch streamingBatch) {
- final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
- final IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeInstance.getDescriptor());
- long start = System.currentTimeMillis();
-
- final Map<Long, HyperLogLogPlusCounter> samplingResult = CubingUtils.sampling(cubeInstance.getDescriptor(), flatDesc, Lists.transform(streamingBatch.getMessages(), new Function<StreamingMessage, List<String>>() {
- @Nullable
- @Override
- public List<String> apply(@Nullable StreamingMessage input) {
- return input.getData();
- }
- }));
- logger.info(String.format("sampling of %d messages cost %d ms", streamingBatch.getMessages().size(), (System.currentTimeMillis() - start)));
- return samplingResult;
- }
-
- @Override
- public Map<TblColRef, Dictionary<String>> buildDictionary(StreamingBatch streamingBatch, IBuildable buildable) {
- final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
- final Map<TblColRef, Dictionary<String>> dictionaryMap;
- try {
- dictionaryMap = CubingUtils.buildDictionary(cubeInstance, Lists.transform(streamingBatch.getMessages(), new Function<StreamingMessage, List<String>>() {
- @Nullable
- @Override
- public List<String> apply(@Nullable StreamingMessage input) {
- return input.getData();
- }
- }));
- Map<TblColRef, Dictionary<String>> realDictMap = CubingUtils.writeDictionary((CubeSegment) buildable, dictionaryMap, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond());
- return realDictMap;
- } catch (IOException e) {
- throw new RuntimeException("failed to build dictionary", e);
- }
- }
-
- @Override
- public void commit(IBuildable buildable) {
- CubeSegment cubeSegment = (CubeSegment) buildable;
- cubeSegment.setStatus(SegmentStatusEnum.READY);
- cubeSegment.setInputRecords(processedRowCount);
- CubeUpdate cubeBuilder = new CubeUpdate(cubeSegment.getCubeInstance());
- cubeBuilder.setToUpdateSegs(cubeSegment);
- try {
- CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).updateCube(cubeBuilder);
- } catch (IOException e) {
- throw new RuntimeException("failed to update CubeSegment", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/diagnose/StreamingLogAnalyzer.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/diagnose/StreamingLogAnalyzer.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/diagnose/StreamingLogAnalyzer.java
deleted file mode 100644
index fba664d..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/diagnose/StreamingLogAnalyzer.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.streaming.diagnose;
-
-import java.io.File;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.io.FileUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class StreamingLogAnalyzer {
- public static void main(String[] args) {
- int errorFileCount = 0;
- List<Long> ellapsedTimes = Lists.newArrayList();
-
- String patternStr = "(\\d{2}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})";
- Pattern pattern = Pattern.compile(patternStr);
-
- SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
-
- Preconditions.checkArgument(args.length == 1, "Usage: StreamingLogsAnalyser streaming_logs_folder");
- for (File file : FileUtils.listFiles(new File(args[0]), new String[] { "log" }, false)) {
- System.out.println("Processing file " + file.toString());
-
- long startTime = 0;
- long endTime = 0;
- try {
- List<String> contents = Files.readAllLines(file.toPath(), Charset.defaultCharset());
- for (int i = 0; i < contents.size(); ++i) {
- Matcher m = pattern.matcher(contents.get(i));
- if (m.find()) {
- startTime = format.parse("20" + m.group(1)).getTime();
- break;
- }
- }
-
- for (int i = contents.size() - 1; i >= 0; --i) {
- Matcher m = pattern.matcher(contents.get(i));
- if (m.find()) {
- endTime = format.parse("20" + m.group(1)).getTime();
- break;
- }
- }
-
- if (startTime == 0 || endTime == 0) {
- throw new RuntimeException("start time or end time is not found");
- }
-
- if (endTime - startTime < 60000) {
- System.out.println("Warning: this job took less than one minute!!!! " + file.toString());
- }
-
- ellapsedTimes.add(endTime - startTime);
-
- } catch (Exception e) {
- System.out.println("Exception when processing log file " + file.toString());
- System.out.println(e);
- errorFileCount++;
- }
- }
-
- System.out.println("Totally error files count " + errorFileCount);
- System.out.println("Totally normal files processed " + ellapsedTimes.size());
-
- long sum = 0;
- for (Long x : ellapsedTimes) {
- sum += x;
- }
- System.out.println("Avg build time " + (sum / ellapsedTimes.size()));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
deleted file mode 100644
index 55252c4..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.streaming.monitor;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.MailService;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class StreamingMonitor {
-
- private static final Logger logger = LoggerFactory.getLogger(StreamingMonitor.class);
-
- public void checkCountAll(List<String> receivers, String host, String authorization, String projectName, String tableName) {
- String title = "checkCountAll job(host:" + host + " tableName:" + tableName + ") ";
- StringBuilder stringBuilder = new StringBuilder();
- String url = host + "/kylin/api/query";
- PostMethod request = new PostMethod(url);
- try {
-
- request.addRequestHeader("Authorization", "Basic " + authorization);
- request.addRequestHeader("Content-Type", "application/json");
- String query = String.format("{\"sql\":\"select count(*) from %s\",\"offset\":0,\"limit\":50000,\"acceptPartial\":true,\"project\":\"%s\"}", tableName, projectName);
- request.setRequestEntity(new ByteArrayRequestEntity(query.getBytes()));
-
- int statusCode = new HttpClient().executeMethod(request);
- String msg = Bytes.toString(request.getResponseBody());
- stringBuilder.append("host:").append(host).append("\n");
- stringBuilder.append("query:").append(query).append("\n");
- stringBuilder.append("statusCode:").append(statusCode).append("\n");
- if (statusCode == 200) {
- title += "succeed";
- final HashMap<?, ?> hashMap = JsonUtil.readValue(msg, HashMap.class);
- stringBuilder.append("results:").append(hashMap.get("results").toString()).append("\n");
- stringBuilder.append("duration:").append(hashMap.get("duration").toString()).append("\n");
- } else {
- title += "failed";
- stringBuilder.append("response:").append(msg).append("\n");
- }
- } catch (Exception e) {
- final StringWriter out = new StringWriter();
- e.printStackTrace(new PrintWriter(out));
- title += "failed";
- stringBuilder.append(out.toString());
- } finally {
- request.releaseConnection();
- }
- logger.info("title:" + title);
- logger.info("content:" + stringBuilder.toString());
- sendMail(receivers, title, stringBuilder.toString());
- }
-
- public static final List<Pair<Long, Long>> findGaps(String cubeName) {
- List<CubeSegment> segments = getSortedReadySegments(cubeName);
- List<Pair<Long, Long>> gaps = Lists.newArrayList();
- for (int i = 0; i < segments.size() - 1; ++i) {
- CubeSegment first = segments.get(i);
- CubeSegment second = segments.get(i + 1);
- if (first.getDateRangeEnd() == second.getDateRangeStart()) {
- continue;
- } else if (first.getDateRangeEnd() < second.getDateRangeStart()) {
- gaps.add(Pair.newPair(first.getDateRangeEnd(), second.getDateRangeStart()));
- }
- }
- return gaps;
- }
-
- private static List<CubeSegment> getSortedReadySegments(String cubeName) {
- final CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
- Preconditions.checkNotNull(cube);
- final List<CubeSegment> segments = cube.getSegments(SegmentStatusEnum.READY);
- logger.info("totally " + segments.size() + " cubeSegments");
- Collections.sort(segments);
- return segments;
- }
-
- public static final List<Pair<String, String>> findOverlaps(String cubeName) {
- List<CubeSegment> segments = getSortedReadySegments(cubeName);
- List<Pair<String, String>> overlaps = Lists.newArrayList();
- for (int i = 0; i < segments.size() - 1; ++i) {
- CubeSegment first = segments.get(i);
- CubeSegment second = segments.get(i + 1);
- if (first.getDateRangeEnd() == second.getDateRangeStart()) {
- continue;
- } else {
- overlaps.add(Pair.newPair(first.getName(), second.getName()));
- }
- }
- return overlaps;
- }
-
- public void checkCube(List<String> receivers, String cubeName, String host) {
- final CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
- if (cube == null) {
- logger.info("cube:" + cubeName + " does not exist");
- return;
- }
- List<Pair<Long, Long>> gaps = findGaps(cubeName);
- List<Pair<String, String>> overlaps = Lists.newArrayList();
- StringBuilder content = new StringBuilder();
- if (!gaps.isEmpty()) {
- content.append("all gaps:").append("\n").append(StringUtils.join(Lists.transform(gaps, new Function<Pair<Long, Long>, String>() {
- @Nullable
- @Override
- public String apply(Pair<Long, Long> input) {
- return parseInterval(input);
- }
- }), "\n")).append("\n");
- }
- if (!overlaps.isEmpty()) {
- content.append("all overlaps:").append("\n").append(StringUtils.join(overlaps, "\n")).append("\n");
- }
- if (content.length() > 0) {
- logger.info(content.toString());
- sendMail(receivers, String.format("%s has gaps or overlaps on host %s", cubeName, host), content.toString());
- } else {
- logger.info("no gaps or overlaps");
- }
- }
-
- private String parseInterval(Pair<Long, Long> interval) {
- return String.format("{%d(%s), %d(%s)}", interval.getFirst(), new Date(interval.getFirst()).toString(), interval.getSecond(), new Date(interval.getSecond()).toString());
- }
-
- private void sendMail(List<String> receivers, String title, String content) {
- final MailService mailService = new MailService(KylinConfig.getInstanceFromEnv());
- mailService.sendMail(receivers, title, content, false);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
deleted file mode 100644
index 5790bc1..0000000
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming.util;
-
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.engine.streaming.IStreamingInput;
-import org.apache.kylin.engine.streaming.IStreamingOutput;
-import org.apache.kylin.engine.streaming.StreamingBatchBuilder;
-import org.apache.kylin.engine.streaming.cube.StreamingCubeBuilder;
-import org.apache.kylin.metadata.realization.RealizationType;
-
-import com.google.common.base.Preconditions;
-
-/**
- * TODO: like MRUtil, use Factory pattern to allow config
- */
-public class StreamingUtils {
-
- public static IStreamingInput getStreamingInput() {
- return (IStreamingInput) ClassUtil.newInstance("org.apache.kylin.source.kafka.KafkaStreamingInput");
- }
-
- public static IStreamingOutput getStreamingOutput() {
- return (IStreamingOutput) ClassUtil.newInstance("org.apache.kylin.storage.hbase.steps.HBaseStreamingOutput");
- }
-
- public static StreamingBatchBuilder getMicroBatchBuilder(RealizationType realizationType, String realizationName) {
- Preconditions.checkNotNull(realizationName);
- if (realizationType == RealizationType.CUBE) {
- return new StreamingCubeBuilder(realizationName);
- } else {
- throw new UnsupportedOperationException("not implemented yet");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index c30abc0..a47fcde 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -44,8 +44,8 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.engine.EngineFactory;
-import org.apache.kylin.engine.streaming.StreamingConfig;
-import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
+import org.apache.kylin.metadata.streaming.StreamingManager;
import org.apache.kylin.job.DeployUtil;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 09ef0e8..72e4069 100644
--- a/pom.xml
+++ b/pom.xml
@@ -225,11 +225,6 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-streaming</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-spark</artifactId>
<version>${project.version}</version>
</dependency>
@@ -1017,7 +1012,6 @@
<module>core-job</module>
<module>core-storage</module>
<module>engine-mr</module>
- <module>engine-streaming</module>
<module>engine-spark</module>
<module>source-hive</module>
<module>source-kafka</module>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index f3374c3..a5fb874 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -26,7 +26,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.exception.BadRequestException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index c4af5f4..34cc57f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -32,7 +32,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index abf0638..170c395 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -29,7 +29,7 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.metadata.streaming.StreamingManager;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.Output;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
index e49e882..7310d9c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.springframework.beans.factory.annotation.Autowired;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index d4cdfd5..e2100c4 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -43,12 +43,6 @@
<artifactId>kylin-core-common</artifactId>
</dependency>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-streaming</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/ByteBufferBackedInputStream.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/ByteBufferBackedInputStream.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/ByteBufferBackedInputStream.java
deleted file mode 100644
index 6981096..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/ByteBufferBackedInputStream.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.kafka;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-
-/**
- */
-class ByteBufferBackedInputStream extends InputStream {
-
- private ByteBuffer buf;
-
- public ByteBufferBackedInputStream(ByteBuffer buf) {
- this.buf = buf;
- }
-
- @Override
- public int read() throws IOException {
- if (!buf.hasRemaining()) {
- return -1;
- }
- return buf.get() & 0xFF;
- }
-
- @Override
- public int read(byte[] bytes, int off, int len) throws IOException {
- if (!buf.hasRemaining()) {
- return -1;
- }
-
- len = Math.min(len, buf.remaining());
- buf.get(bytes, off, len);
- return len;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index d039583..208c0ce 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
@@ -20,7 +20,7 @@ package org.apache.kylin.source.kafka;
import com.google.common.collect.Lists;
import org.apache.kylin.engine.mr.IMRInput;
-import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.ReadableTable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
deleted file mode 100644
index 78a67c2..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.kafka;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import com.google.common.base.Function;
-import kafka.cluster.BrokerEndPoint;
-import org.apache.kafka.common.protocol.SecurityProtocol;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.engine.EngineFactory;
-import org.apache.kylin.engine.streaming.IStreamingInput;
-import org.apache.kylin.engine.streaming.StreamingConfig;
-import org.apache.kylin.engine.streaming.StreamingManager;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-import org.apache.kylin.source.kafka.util.KafkaRequester;
-import org.apache.kylin.source.kafka.util.KafkaUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-import kafka.cluster.Broker;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.PartitionMetadata;
-import kafka.message.MessageAndOffset;
-
-import javax.annotation.Nullable;
-
-@SuppressWarnings("unused")
-public class KafkaStreamingInput implements IStreamingInput {
-
- private static final Logger logger = LoggerFactory.getLogger(KafkaStreamingInput.class);
-
- @Override
- public StreamingBatch getBatchWithTimeWindow(RealizationType realizationType, String realizationName, int id, long startTime, long endTime) {
- if (realizationType != RealizationType.CUBE) {
- throw new IllegalArgumentException("Unsupported realization in KafkaStreamingInput: " + realizationType);
- }
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- final CubeInstance cube = CubeManager.getInstance(kylinConfig).getCube(realizationName);
- final String streaming = cube.getFactTable();
- final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
- final StreamingConfig streamingConfig = streamingManager.getConfig(streaming);
- if (streamingConfig == null) {
- throw new IllegalArgumentException("Table " + streaming + " is not a streaming table.");
- }
- if (StreamingConfig.STREAMING_TYPE_KAFKA.equals(streamingConfig.getType())) {
- logger.info(String.format("prepare to get streaming batch, name:%s, id:%d, startTime:%d, endTime:%d", streaming, id, startTime, endTime));
-
- try {
- final KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
- final KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(streaming);
- List<TblColRef> columns = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor()).getAllColumns();
-
- final StreamingParser streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getParserProperties(), columns);
- final ExecutorService executorService = Executors.newCachedThreadPool();
- final List<Future<List<StreamingMessage>>> futures = Lists.newArrayList();
- for (final KafkaClusterConfig kafkaClusterConfig : kafkaConfig.getKafkaClusterConfigs()) {
-
- final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
- for (int i = 0; i < partitionCount; ++i) {
- final StreamingMessageProducer producer = new StreamingMessageProducer(kafkaClusterConfig, i, Pair.newPair(startTime, endTime), kafkaConfig.getMargin(), streamingParser);
- final Future<List<StreamingMessage>> future = executorService.submit(producer);
- futures.add(future);
- }
- }
- List<StreamingMessage> messages = Lists.newLinkedList();
- for (Future<List<StreamingMessage>> future : futures) {
- try {
- messages.addAll(future.get());
- } catch (InterruptedException e) {
- logger.warn("this thread should not be interrupted, just ignore", e);
- continue;
- } catch (ExecutionException e) {
- throw new RuntimeException("error when get StreamingMessages", e.getCause());
- }
- }
- final Pair<Long, Long> timeRange = Pair.newPair(startTime, endTime);
- logger.info("finish to get streaming batch, total message count:" + messages.size());
- return new StreamingBatch(messages, timeRange);
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException("failed to create instance of StreamingParser", e);
- }
- } else {
- throw new IllegalArgumentException("kafka is the only supported streaming type.");
- }
- }
-
- private static class StreamingMessageProducer implements Callable<List<StreamingMessage>> {
-
- private final KafkaClusterConfig kafkaClusterConfig;
- private final int partitionId;
- private final StreamingParser streamingParser;
- private final Pair<Long, Long> timeRange;
- private final long margin;
-
- private List<Broker> replicaBrokers;
-
- StreamingMessageProducer(KafkaClusterConfig kafkaClusterConfig, int partitionId, Pair<Long, Long> timeRange, long margin, StreamingParser streamingParser) {
- this.kafkaClusterConfig = kafkaClusterConfig;
- this.partitionId = partitionId;
- this.streamingParser = streamingParser;
- this.margin = margin;
- this.timeRange = timeRange;
- this.replicaBrokers = kafkaClusterConfig.getBrokers();
- }
-
- private Broker getLeadBroker() {
- final PartitionMetadata partitionMetadata = KafkaRequester.getPartitionMetadata(kafkaClusterConfig.getTopic(), partitionId, replicaBrokers, kafkaClusterConfig);
- if (partitionMetadata != null) {
- if (partitionMetadata.errorCode() != 0) {
- logger.warn("PartitionMetadata errorCode: " + partitionMetadata.errorCode());
- }
- replicaBrokers = Lists.transform(partitionMetadata.replicas(), new Function<BrokerEndPoint, Broker>() {
- @Nullable
- @Override
- public Broker apply(@Nullable BrokerEndPoint brokerEndPoint) {
- return new Broker(brokerEndPoint, SecurityProtocol.PLAINTEXT);
- }
- });
- BrokerEndPoint leaderEndpoint = partitionMetadata.leader();
-
- return new Broker(leaderEndpoint, SecurityProtocol.PLAINTEXT);
- } else {
- return null;
- }
- }
-
- @Override
- public List<StreamingMessage> call() throws Exception {
- List<StreamingMessage> result = Lists.newLinkedList();
- try {
- long startTimestamp = timeRange.getFirst() - margin;
- long offset = KafkaUtils.findClosestOffsetWithDataTimestamp(kafkaClusterConfig, partitionId, startTimestamp, streamingParser);
- int fetchRound = 0;
- int consumeMsgCount = 0;
- Broker leadBroker = null;
- String topic = kafkaClusterConfig.getTopic();
- while (true) {
- boolean outOfMargin = false;
- int consumeMsgCountAtBeginning = consumeMsgCount;
- fetchRound++;
-
- if (leadBroker == null) {
- leadBroker = getLeadBroker();
- }
-
- if (leadBroker == null) {
- logger.warn("cannot find lead broker, wait 5s");
- Thread.sleep(5000);
- continue;
- }
-
- logger.info("fetching topic {} partition id {} offset {} leader {}", topic, String.valueOf(partitionId), String.valueOf(offset), leadBroker.toString());
-
- final FetchResponse fetchResponse = KafkaRequester.fetchResponse(topic, partitionId, offset, leadBroker, kafkaClusterConfig);
- if (fetchResponse.errorCode(topic, partitionId) != 0) {
- logger.warn("fetch response offset:" + offset + " errorCode:" + fetchResponse.errorCode(topic, partitionId));
- Thread.sleep(30000);
- continue;
- }
-
- for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topic, partitionId)) {
- offset++;
- consumeMsgCount++;
- final StreamingMessage streamingMessage = streamingParser.parse(messageAndOffset.message().payload());
- streamingMessage.setOffset(messageAndOffset.offset());
- if (streamingParser.filter(streamingMessage)) {
- final long timestamp = streamingMessage.getTimestamp();
- if (timestamp >= timeRange.getFirst() && timestamp < timeRange.getSecond()) {
- result.add(streamingMessage);
- } else if (timestamp < timeRange.getSecond() + margin) {
- //do nothing
- } else {
- logger.info("thread:" + Thread.currentThread() + " message timestamp:" + timestamp + " is out of time range:" + timeRange + " margin:" + margin);
- outOfMargin = true;
- break;
- }
- }
- }
- logger.info("Number of messages consumed: " + consumeMsgCount + " offset is: " + offset + " total fetch round: " + fetchRound);
- if (outOfMargin) {
- break;
- }
- if (consumeMsgCount == consumeMsgCountAtBeginning) {//nothing this round
- logger.info("no message consumed this round, wait 30s");
- Thread.sleep(30000);
- }
- }
- } catch (InterruptedException e) {
- logger.warn("this thread should not be interrupted, just stop fetching", e);
- }
- return result;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index e4c702d..633a30c 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -33,6 +33,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.StreamingMessage;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.kafka.util.ByteBufferBackedInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaInputAnalyzer.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaInputAnalyzer.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaInputAnalyzer.java
deleted file mode 100644
index b1b4011..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaInputAnalyzer.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.diagnose;
-
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.Maps;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractApplication;
-import org.apache.kylin.common.util.DaemonThreadFactory;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.kafka.KafkaConfigManager;
-import org.apache.kylin.source.kafka.StreamingParser;
-import org.apache.kylin.source.kafka.TimedJsonStreamParser;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-import org.apache.kylin.source.kafka.util.KafkaRequester;
-import org.apache.kylin.source.kafka.util.KafkaUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-import kafka.api.OffsetRequest;
-import kafka.cluster.Broker;
-import kafka.javaapi.FetchResponse;
-import kafka.message.MessageAndOffset;
-
-/**
- * Continuously run this as a daemon to discover how "disordered" the kafka queue is.
- * This daemon only store a digest so it should not be space-consuming
- */
-public class KafkaInputAnalyzer extends AbstractApplication {
-
- public class KafkaMessagePuller implements Runnable {
-
- private final String topic;
- private final int partitionId;
- private final KafkaClusterConfig streamingConfig;
- private final LinkedBlockingQueue<StreamingMessage> streamQueue;
- private final StreamingParser streamingParser;
- private final Broker leadBroker;
- private long offset;
-
- protected final Logger logger;
-
- public KafkaMessagePuller(int clusterID, String topic, int partitionId, long startOffset, Broker leadBroker, KafkaClusterConfig kafkaClusterConfig, StreamingParser streamingParser) {
- this.topic = topic;
- this.partitionId = partitionId;
- this.streamingConfig = kafkaClusterConfig;
- this.offset = startOffset;
- this.logger = LoggerFactory.getLogger(topic + "_cluster_" + clusterID + "_" + partitionId);
- this.streamQueue = new LinkedBlockingQueue<StreamingMessage>(10000);
- this.streamingParser = streamingParser;
- this.leadBroker = leadBroker;
- }
-
- public BlockingQueue<StreamingMessage> getStreamQueue() {
- return streamQueue;
- }
-
- @Override
- public void run() {
- try {
- int consumeMsgCount = 0;
- int fetchRound = 0;
- while (true) {
- int consumeMsgCountAtBeginning = consumeMsgCount;
- fetchRound++;
-
- logger.info("fetching topic {} partition id {} offset {} leader {}", topic, String.valueOf(partitionId), String.valueOf(offset), leadBroker.toString());
-
- final FetchResponse fetchResponse = KafkaRequester.fetchResponse(topic, partitionId, offset, leadBroker, streamingConfig);
- if (fetchResponse.errorCode(topic, partitionId) != 0) {
- logger.warn("fetch response offset:" + offset + " errorCode:" + fetchResponse.errorCode(topic, partitionId));
- Thread.sleep(30000);
- continue;
- }
-
- for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topic, partitionId)) {
- offset++;
- consumeMsgCount++;
-
- final StreamingMessage streamingMessage = streamingParser.parse(messageAndOffset.message().payload());
- streamingMessage.setOffset(messageAndOffset.offset());
- if (streamingParser.filter(streamingMessage)) {
- streamQueue.add(streamingMessage);
- }
-
- }
- logger.info("Number of messages consumed: " + consumeMsgCount + " offset is: " + offset + " total fetch round: " + fetchRound);
-
- if (consumeMsgCount == consumeMsgCountAtBeginning) {//nothing this round
- Thread.sleep(30000);
- }
- }
- } catch (Exception e) {
- logger.error("consumer has encountered an error", e);
- }
- }
-
- }
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_STREAMING = OptionBuilder.withArgName("streaming").hasArg().isRequired(true).withDescription("Name of the streaming").create("streaming");
- @SuppressWarnings("static-access")
- private static final Option OPTION_TASK = OptionBuilder.withArgName("task").hasArg().isRequired(true).withDescription("get delay or get disorder degree").create("task");
- @SuppressWarnings("static-access")
- private static final Option OPTION_TSCOLNAME = OptionBuilder.withArgName("tsColName").hasArg().isRequired(true).withDescription("field name of the ts").create("tsColName");
-
- private static final Logger logger = LoggerFactory.getLogger(KafkaInputAnalyzer.class);
-
- private StreamingParser parser;
- private KafkaConfig kafkaConfig;
-
- private Options options;
-
- public KafkaInputAnalyzer() {
- options = new Options();
- options.addOption(OPTION_STREAMING);
- options.addOption(OPTION_TASK);
- options.addOption(OPTION_TSCOLNAME);
-
- }
-
- private List<BlockingQueue<StreamingMessage>> consume(final int clusterID, final KafkaClusterConfig kafkaClusterConfig, final int partitionCount, long whichtime) {
- List<BlockingQueue<StreamingMessage>> result = Lists.newArrayList();
- for (int partitionId = 0; partitionId < partitionCount; ++partitionId) {
- final kafka.cluster.Broker leadBroker = KafkaUtils.getLeadBroker(kafkaClusterConfig, partitionId);
- long streamingOffset = KafkaRequester.getLastOffset(kafkaClusterConfig.getTopic(), partitionId, whichtime, leadBroker, kafkaClusterConfig);
- logger.info("starting offset:" + streamingOffset + " cluster id:" + clusterID + " partitionId:" + partitionId);
- KafkaMessagePuller consumer = new KafkaMessagePuller(clusterID, kafkaClusterConfig.getTopic(), partitionId, streamingOffset, leadBroker, kafkaClusterConfig, parser);
- Executors.newSingleThreadExecutor(new DaemonThreadFactory()).submit(consumer);
- result.add(consumer.getStreamQueue());
- }
- return result;
- }
-
- private List<BlockingQueue<StreamingMessage>> consumeAll(long whichtime) {
- int clusterId = 0;
- final List<BlockingQueue<StreamingMessage>> queues = Lists.newLinkedList();
-
- for (final KafkaClusterConfig kafkaClusterConfig : kafkaConfig.getKafkaClusterConfigs()) {
- final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
- final List<BlockingQueue<StreamingMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, whichtime);
- queues.addAll(oneClusterQueue);
- logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
- clusterId++;
- }
- return queues;
- }
-
- private void analyzeLatency() throws InterruptedException {
- long[] intervals = new long[] { 1, 5, 60, 300, 1800 };
- final List<BlockingQueue<StreamingMessage>> allPartitionData = consumeAll(OffsetRequest.LatestTime());
- final List<TimeHistogram> allHistograms = Lists.newArrayList();
- final TimeHistogram overallHistogram = new TimeHistogram(intervals, "overall");
-
- ExecutorService executorService = Executors.newFixedThreadPool(allPartitionData.size(), new DaemonThreadFactory());
- for (int i = 0; i < allPartitionData.size(); ++i) {
- final int index = i;
- allHistograms.add(new TimeHistogram(intervals, "" + i));
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- while (true) {
- try {
- StreamingMessage message = allPartitionData.get(index).take();
- long t = message.getTimestamp();
- allHistograms.get(index).processMillis(System.currentTimeMillis() - t);
- overallHistogram.processMillis(System.currentTimeMillis() - t);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- });
- }
-
- while (true) {
- System.out.println("Printing status at : " + new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").format(Calendar.getInstance().getTime()));
-
- for (TimeHistogram histogram : allHistograms) {
- histogram.printStatus();
- }
- overallHistogram.printStatus();
- Thread.sleep(300000);
- }
- }
-
- private void analyzeDisorder() throws InterruptedException {
- final List<BlockingQueue<StreamingMessage>> allPartitionData = consumeAll(OffsetRequest.EarliestTime());
-
- final List<Long> wallClocks = Lists.newArrayList();
- final List<Long> wallOffset = Lists.newArrayList();
- final List<Long> maxDisorderTime = Lists.newArrayList();
- final List<Long> maxDisorderOffset = Lists.newArrayList();
- final List<Long> processedMessages = Lists.newArrayList();
-
- for (int i = 0; i < allPartitionData.size(); i++) {
- wallClocks.add(0L);
- wallOffset.add(0L);
- maxDisorderTime.add(0L);
- maxDisorderOffset.add(0L);
- processedMessages.add(0L);
- }
-
- ExecutorService executorService = Executors.newFixedThreadPool(allPartitionData.size(), new DaemonThreadFactory());
- final CountDownLatch countDownLatch = new CountDownLatch(allPartitionData.size());
- for (int i = 0; i < allPartitionData.size(); ++i) {
- final int index = i;
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- try {
- while (true) {
- StreamingMessage message = allPartitionData.get(index).poll(60, TimeUnit.SECONDS);
- if (message == null) {
- System.out.println(String.format("Thread %d is exiting", index));
- break;
- }
- long t = message.getTimestamp();
- long offset = message.getOffset();
- if (t < wallClocks.get(index)) {
- maxDisorderTime.set(index, Math.max(wallClocks.get(index) - t, maxDisorderTime.get(index)));
- maxDisorderOffset.set(index, Math.max(offset - wallOffset.get(index), maxDisorderOffset.get(index)));
- } else {
- wallClocks.set(index, t);
- wallOffset.set(index, offset);
- }
- processedMessages.set(index, processedMessages.get(index) + 1);
-
- if (processedMessages.get(index) % 10000 == 1) {
- System.out.println(String.format("Thread %d processed %d messages. Max disorder time is %d , max disorder offset is %d", //
- index, processedMessages.get(index), maxDisorderTime.get(index), maxDisorderOffset.get(index)));
- }
- }
-
- System.out.println(String.format("Thread %d finishes after %d messages. Max disorder time is %d , max disorder offset is %d", //
- index, processedMessages.get(index), maxDisorderTime.get(index), maxDisorderOffset.get(index)));
- countDownLatch.countDown();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
- }
-
- countDownLatch.await();
- }
-
- @Override
- protected Options getOptions() {
- return options;
- }
-
- @Override
- protected void execute(OptionsHelper optionsHelper) throws Exception {
-
- String streaming = optionsHelper.getOptionValue(OPTION_STREAMING);
- String task = optionsHelper.getOptionValue(OPTION_TASK);
- String tsColName = optionsHelper.getOptionValue(OPTION_TSCOLNAME);
-
- Map<String, String> properties = Maps.newHashMap();
- properties.put(StreamingParser.PROPERTY_TS_COLUMN_NAME, tsColName);
- kafkaConfig = KafkaConfigManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig(streaming);
- parser = new TimedJsonStreamParser(Lists.<TblColRef> newArrayList(), properties);
-
- if ("disorder".equalsIgnoreCase(task)) {
- analyzeDisorder();
- } else if ("delay".equalsIgnoreCase(task)) {
- analyzeLatency();
- } else {
- optionsHelper.printUsage(this.getClass().getName(), options);
- }
- }
-
- public static void main(String[] args) {
- KafkaInputAnalyzer analyzer = new KafkaInputAnalyzer();
- analyzer.execute(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaVerify.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaVerify.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaVerify.java
deleted file mode 100644
index 6a456bc..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/KafkaVerify.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.diagnose;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.fasterxml.jackson.databind.JavaType;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.type.MapType;
-import com.fasterxml.jackson.databind.type.SimpleType;
-
-/**
- * only for verify kylin streaming's correctness by comparing to data in original kafka topic
- */
-public class KafkaVerify {
-
- public static void main(String[] args) throws IOException {
-
- System.out.println("start");
-
- ObjectMapper mapper = new ObjectMapper();
- JavaType mapType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
-
- long start = Long.valueOf(args[0]);
- long end = Long.valueOf(args[1]);
- long interval = Long.valueOf(args[2]);
- int bucket = (int) ((end - start + interval - 1) / interval);
-
- long[] qtySum = new long[bucket];
- long qtyTotal = 0;
- long[] counts = new long[bucket];
- long countTotal = 0;
- long processed = 0;
- long minOffset = -1;
- long maxOffset = -1;
-
- try (BufferedReader br = new BufferedReader(new FileReader(new File(args[3])))) {
- String s;
- while ((s = br.readLine()) != null) {
- // process the line.
- if (++processed % 10000 == 1) {
- System.out.println("processing " + processed);
- }
-
- Map<String, String> root = mapper.readValue(s, mapType);
- String tsStr = root.get("sys_ts");
-
- if (StringUtils.isEmpty(tsStr)) {
- continue;
- }
- long ts = Long.valueOf(tsStr);
- if (ts < start || ts >= end) {
- continue;
- }
-
- if (minOffset == -1) {
- minOffset = processed - 1;
- }
- maxOffset = processed - 1;
-
- long qty = Long.valueOf(root.get("qty"));
- int index = (int) ((ts - start) / interval);
- qtySum[index] += qty;
- qtyTotal += qty;
- counts[index]++;
- countTotal++;
- }
- }
-
- System.out.println("qty sum is " + Arrays.toString(qtySum));
- System.out.println("qty total is " + qtyTotal);
- System.out.println("count is " + Arrays.toString(counts));
- System.out.println("count total is " + countTotal);
- System.out.println("first processed is " + minOffset);
- System.out.println("last processed is " + maxOffset);
- }
-}
[25/50] [abbrv] kylin git commit: minor,
move timezone setting early to surefire command line
Posted by li...@apache.org.
minor, move timezone setting early to surefire command line
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61a08d42
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61a08d42
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61a08d42
Branch: refs/heads/master-cdh5.7
Commit: 61a08d429a2c7419751b312e41bcee82048716e2
Parents: 021ccfd
Author: Yang Li <li...@apache.org>
Authored: Sun Oct 16 07:59:33 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Oct 16 07:59:33 2016 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 3 ---
pom.xml | 2 +-
2 files changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/61a08d42/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 93d47f1..59a3a04 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -24,7 +24,6 @@ import java.io.File;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
-import java.util.TimeZone;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -66,8 +65,6 @@ public class ITKylinQueryTest extends KylinTestBase {
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
-
- TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/kylin/blob/61a08d42/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index caa09ec..c7538e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,7 +124,7 @@
</sonar.jacoco.excludes>
<!-- JVM Args for Testing -->
- <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine>
+ <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M -Duser.timezone=UTC</argLine>
</properties>
<licenses>