You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/07/01 10:43:00 UTC
[1/4] incubator-kylin git commit: KYLIN-864 half way
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 3d83a253a -> 098f5800d
KYLIN-864 half way
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/098f5800
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/098f5800
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/098f5800
Branch: refs/heads/0.8
Commit: 098f5800db31ef75d00eb50f762bd78ecf73ca76
Parents: fa8aa12
Author: honma <ho...@ebay.com>
Authored: Wed Jul 1 16:42:15 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800
----------------------------------------------------------------------
.../kylin/job/streaming/StreamingBootstrap.java | 83 +++++-----
.../kylin/streaming/MarginCalculator.java | 166 +++++++++----------
2 files changed, 122 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/098f5800/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 864062f..d4c5139 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,24 +34,17 @@
package org.apache.kylin.job.streaming;
-import java.lang.reflect.Constructor;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-import javax.annotation.Nullable;
-
-import jodd.io.StreamUtil;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import kafka.api.OffsetRequest;
import kafka.cluster.Broker;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.util.DaemonThreadFactory;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -67,10 +60,12 @@ import org.apache.kylin.streaming.invertedindex.IIStreamConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import javax.annotation.Nullable;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
/**
*/
@@ -277,35 +272,35 @@ public class StreamingBootstrap {
logger.info("one off build finished");
}
- private void startCalculatingMargin(final StreamingConfig streamingConfig) throws Exception {
- final String cubeName = streamingConfig.getCubeName();
- final StreamParser streamParser = getStreamParser(streamingConfig, Lists.<TblColRef>newArrayList());
- final List<BlockingQueue<StreamMessage>> queues = Lists.newLinkedList();
-
- int clusterId = 0;
- final List<Pair<Long,Long>> firstAndLastOffsets = Lists.newArrayList();
-
- for (final KafkaClusterConfig kafkaClusterConfig : streamingConfig.getKafkaClusterConfigs()) {
- final ConcurrentMap<Integer, Long> partitionIdOffsetMap = Maps.newConcurrentMap();
- final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
- for (int i = 0; i < partitionCount; ++i) {
- Pair<Long,Long> firstlast = StreamingUtil.getFirstAndLastOffset(kafkaClusterConfig,i);
- firstAndLastOffsets.add(firstlast);
- partitionIdOffsetMap.putIfAbsent(i,firstlast.getFirst());
- }
-
- logger.info("partitionId to start offset map:" + partitionIdOffsetMap);
- Preconditions.checkArgument(partitionIdOffsetMap.size() == partitionCount, "fail to get all start offset");
- final List<BlockingQueue<StreamMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, partitionIdOffsetMap, 0);
- queues.addAll(oneClusterQueue);
- logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
- clusterId++;
- }
-
- OneOffStreamBuilder oneOffStreamBuilder = new OneOffStreamBuilder(streamingConfig.getName(), queues, streamParser, new CubeStreamConsumer(cubeName), startTimestamp, endTimestamp, margin);
- Executors.newSingleThreadExecutor().submit(oneOffStreamBuilder).get();
- logger.info("one off build finished");
- }
+// private void startCalculatingMargin(final StreamingConfig streamingConfig) throws Exception {
+// final String cubeName = streamingConfig.getCubeName();
+// final StreamParser streamParser = getStreamParser(streamingConfig, Lists.<TblColRef>newArrayList());
+// final List<BlockingQueue<StreamMessage>> queues = Lists.newLinkedList();
+//
+// int clusterId = 0;
+// final List<Pair<Long,Long>> firstAndLastOffsets = Lists.newArrayList();
+//
+// for (final KafkaClusterConfig kafkaClusterConfig : streamingConfig.getKafkaClusterConfigs()) {
+// final ConcurrentMap<Integer, Long> partitionIdOffsetMap = Maps.newConcurrentMap();
+// final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
+// for (int i = 0; i < partitionCount; ++i) {
+// Pair<Long,Long> firstlast = StreamingUtil.getFirstAndLastOffset(kafkaClusterConfig,i);
+// firstAndLastOffsets.add(firstlast);
+// partitionIdOffsetMap.putIfAbsent(i,firstlast.getFirst());
+// }
+//
+// logger.info("partitionId to start offset map:" + partitionIdOffsetMap);
+// Preconditions.checkArgument(partitionIdOffsetMap.size() == partitionCount, "fail to get all start offset");
+// final List<BlockingQueue<StreamMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, partitionIdOffsetMap, 0);
+// queues.addAll(oneClusterQueue);
+// logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
+// clusterId++;
+// }
+//
+// OneOffStreamBuilder oneOffStreamBuilder = new OneOffStreamBuilder(streamingConfig.getName(), queues, streamParser, new CubeStreamConsumer(cubeName), startTimestamp, endTimestamp, margin);
+// Executors.newSingleThreadExecutor().submit(oneOffStreamBuilder).get();
+// logger.info("one off build finished");
+// }
private void startIIStreaming(StreamingConfig streamingConfig, final int partitionId) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/098f5800/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
index dc18cc4..855a0f0 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
@@ -1,83 +1,83 @@
-package org.apache.kylin.streaming;
-
-import java.util.List;
-import java.util.concurrent.*;
-
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class MarginCalculator implements Runnable {
-
- private static final Logger logger = LoggerFactory.getLogger(MarginCalculator.class);
-
-
- private final String streaming;
- private final List<BlockingQueue<StreamMessage>> queues;
- private final List<Pair<Long,Long>> firstAndLastOffsets;
- private StreamParser streamParser;
-
- public MarginCalculator(String streaming, List<BlockingQueue<StreamMessage>> queues, StreamParser streamParser, List<Pair<Long,Long>> firstAndLastOffsets) {
- Preconditions.checkArgument(queues.size() > 0);
- this.streaming = streaming;
- this.queues = queues;
- this.streamParser = streamParser;
- this.firstAndLastOffsets = firstAndLastOffsets;
- }
-
- @Override
- public void run() {
- try {
- final int inputCount = queues.size();
- final ExecutorService executorService = Executors.newFixedThreadPool(inputCount);
- final CountDownLatch countDownLatch = new CountDownLatch(inputCount);
- final List<Future<MicroStreamBatch>> futures = Lists.newLinkedList();
- int partitionId = 0;
- for (BlockingQueue<StreamMessage> queue : queues) {
- futures.add(executorService.submit(new StreamFetcher(partitionId, queue, countDownLatch, batchCondition, streamParser)));
- }
- countDownLatch.await();
- List<MicroStreamBatch> batches = Lists.newLinkedList();
- for (Future<MicroStreamBatch> future : futures) {
- if (future.get() != null) {
- batches.add(future.get());
- } else {
- logger.warn("EOF encountered, stop streaming");
- }
- }
-
- MicroStreamBatch batch = batches.get(0);
- if (batches.size() > 1) {
- for (int i = 1; i < inputCount; i++) {
- if (batches.get(i).size() > 0) {
- batch = MicroStreamBatch.union(batch, batches.get(i));
- }
- }
- }
- batch.getTimestamp().setFirst(batchCondition.getStartTime());
- batch.getTimestamp().setSecond(batchCondition.getEndTime());
-
- logger.info("Consuming {} messages, covering from {} to {}", new String[] { String.valueOf(batch.size()), DateFormat.formatToTimeStr(batch.getTimestamp().getFirst()), DateFormat.formatToTimeStr(batch.getTimestamp().getSecond()) });
- long startTime = System.currentTimeMillis();
- consumer.consume(batch);
- logger.info("Batch build costs {} milliseconds", System.currentTimeMillis() - startTime);
- } catch (InterruptedException ie) {
- throw new RuntimeException("this thread should not be interrupted", ie);
- } catch (ExecutionException ee) {
- logger.error("fetch stream error", ee);
- throw new RuntimeException(ee);
- } catch (Exception e) {
- logger.error("error consume batch", e);
- throw new RuntimeException("error consume batch", e);
- }
-
-
- }
-
-}
+//package org.apache.kylin.streaming;
+//
+//import java.util.List;
+//import java.util.concurrent.*;
+//
+//import org.apache.kylin.common.util.DateFormat;
+//import org.apache.kylin.common.util.Pair;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//
+//import com.google.common.base.Preconditions;
+//import com.google.common.collect.Lists;
+//
+///**
+// */
+//public class MarginCalculator implements Runnable {
+//
+// private static final Logger logger = LoggerFactory.getLogger(MarginCalculator.class);
+//
+//
+// private final String streaming;
+// private final List<BlockingQueue<StreamMessage>> queues;
+// private final List<Pair<Long,Long>> firstAndLastOffsets;
+// private StreamParser streamParser;
+//
+// public MarginCalculator(String streaming, List<BlockingQueue<StreamMessage>> queues, StreamParser streamParser, List<Pair<Long,Long>> firstAndLastOffsets) {
+// Preconditions.checkArgument(queues.size() > 0);
+// this.streaming = streaming;
+// this.queues = queues;
+// this.streamParser = streamParser;
+// this.firstAndLastOffsets = firstAndLastOffsets;
+// }
+//
+// @Override
+// public void run() {
+// try {
+// final int inputCount = queues.size();
+// final ExecutorService executorService = Executors.newFixedThreadPool(inputCount);
+// final CountDownLatch countDownLatch = new CountDownLatch(inputCount);
+// final List<Future<MicroStreamBatch>> futures = Lists.newLinkedList();
+// int partitionId = 0;
+// for (BlockingQueue<StreamMessage> queue : queues) {
+// futures.add(executorService.submit(new StreamFetcher(partitionId, queue, countDownLatch, batchCondition, streamParser)));
+// }
+// countDownLatch.await();
+// List<MicroStreamBatch> batches = Lists.newLinkedList();
+// for (Future<MicroStreamBatch> future : futures) {
+// if (future.get() != null) {
+// batches.add(future.get());
+// } else {
+// logger.warn("EOF encountered, stop streaming");
+// }
+// }
+//
+// MicroStreamBatch batch = batches.get(0);
+// if (batches.size() > 1) {
+// for (int i = 1; i < inputCount; i++) {
+// if (batches.get(i).size() > 0) {
+// batch = MicroStreamBatch.union(batch, batches.get(i));
+// }
+// }
+// }
+// batch.getTimestamp().setFirst(batchCondition.getStartTime());
+// batch.getTimestamp().setSecond(batchCondition.getEndTime());
+//
+// logger.info("Consuming {} messages, covering from {} to {}", new String[] { String.valueOf(batch.size()), DateFormat.formatToTimeStr(batch.getTimestamp().getFirst()), DateFormat.formatToTimeStr(batch.getTimestamp().getSecond()) });
+// long startTime = System.currentTimeMillis();
+// consumer.consume(batch);
+// logger.info("Batch build costs {} milliseconds", System.currentTimeMillis() - startTime);
+// } catch (InterruptedException ie) {
+// throw new RuntimeException("this thread should not be interrupted", ie);
+// } catch (ExecutionException ee) {
+// logger.error("fetch stream error", ee);
+// throw new RuntimeException(ee);
+// } catch (Exception e) {
+// logger.error("error consume batch", e);
+// throw new RuntimeException("error consume batch", e);
+// }
+//
+//
+// }
+//
+//}
[4/4] incubator-kylin git commit: KYLIN-864
Posted by ma...@apache.org.
KYLIN-864
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1657db8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1657db8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1657db8f
Branch: refs/heads/0.8
Commit: 1657db8f50c641496fabc8d4424ed9b62fd16d33
Parents: 3d83a25
Author: honma <ho...@ebay.com>
Authored: Tue Jun 30 16:30:49 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 12 +--
.../kylin/job/streaming/BootstrapConfig.java | 1 +
.../kylin/job/streaming/StreamingBootstrap.java | 55 ++++++++--
.../kylin/job/streaming/StreamingCLI.java | 43 ++++----
.../observer/AggregateRegionObserver.java | 13 ++-
.../kylin/streaming/MarginCalculator.java | 83 +++++++++++++++
.../kylin/streaming/OffsetPeriodCondition.java | 35 +++++++
.../apache/kylin/streaming/PartitionMargin.java | 13 +++
.../streaming/PartitionMarginCalculator.java | 103 +++++++++++++++++++
.../apache/kylin/streaming/StreamFetcher.java | 2 +-
.../apache/kylin/streaming/StreamingUtil.java | 24 ++++-
.../kylin/streaming/TimedJsonStreamParser.java | 5 -
12 files changed, 337 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 5ce76aa..560a836 100644
--- a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -64,9 +64,9 @@ public class BackdoorToggles {
*
*
*
- example:
+ example:(put it into request body)
"backdoorToggles": {
- "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+ "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
}
*/
public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
@@ -74,9 +74,9 @@ public class BackdoorToggles {
/**
* set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
*
- example:
+ example:(put it into request body)
"backdoorToggles": {
- "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
}
*/
public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
@@ -84,9 +84,9 @@ public class BackdoorToggles {
/**
* set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
*
- example:
+ example:(put it into request body)
"backdoorToggles": {
- "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
+ "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
}
*/
public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
index a82fec3..302f455 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
@@ -12,6 +12,7 @@ public class BootstrapConfig {
private long end = 0L;
private long margin = 0L;
+
public long getMargin() {
return margin;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 98bbcd4..864062f 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,17 +34,24 @@
package org.apache.kylin.job.streaming;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+import javax.annotation.Nullable;
+
+import jodd.io.StreamUtil;
import kafka.api.OffsetRequest;
import kafka.cluster.Broker;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.util.DaemonThreadFactory;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -60,12 +67,10 @@ import org.apache.kylin.streaming.invertedindex.IIStreamConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
-import java.lang.reflect.Constructor;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
*/
@@ -272,6 +277,36 @@ public class StreamingBootstrap {
logger.info("one off build finished");
}
+ private void startCalculatingMargin(final StreamingConfig streamingConfig) throws Exception {
+ final String cubeName = streamingConfig.getCubeName();
+ final StreamParser streamParser = getStreamParser(streamingConfig, Lists.<TblColRef>newArrayList());
+ final List<BlockingQueue<StreamMessage>> queues = Lists.newLinkedList();
+
+ int clusterId = 0;
+ final List<Pair<Long,Long>> firstAndLastOffsets = Lists.newArrayList();
+
+ for (final KafkaClusterConfig kafkaClusterConfig : streamingConfig.getKafkaClusterConfigs()) {
+ final ConcurrentMap<Integer, Long> partitionIdOffsetMap = Maps.newConcurrentMap();
+ final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
+ for (int i = 0; i < partitionCount; ++i) {
+ Pair<Long,Long> firstlast = StreamingUtil.getFirstAndLastOffset(kafkaClusterConfig,i);
+ firstAndLastOffsets.add(firstlast);
+ partitionIdOffsetMap.putIfAbsent(i,firstlast.getFirst());
+ }
+
+ logger.info("partitionId to start offset map:" + partitionIdOffsetMap);
+ Preconditions.checkArgument(partitionIdOffsetMap.size() == partitionCount, "fail to get all start offset");
+ final List<BlockingQueue<StreamMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, partitionIdOffsetMap, 0);
+ queues.addAll(oneClusterQueue);
+ logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
+ clusterId++;
+ }
+
+ OneOffStreamBuilder oneOffStreamBuilder = new OneOffStreamBuilder(streamingConfig.getName(), queues, streamParser, new CubeStreamConsumer(cubeName), startTimestamp, endTimestamp, margin);
+ Executors.newSingleThreadExecutor().submit(oneOffStreamBuilder).get();
+ logger.info("one off build finished");
+ }
+
private void startIIStreaming(StreamingConfig streamingConfig, final int partitionId) throws Exception {
List<KafkaClusterConfig> allClustersConfigs = streamingConfig.getKafkaClusterConfigs();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index 409c558..1eb23f1 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -34,7 +34,6 @@
package org.apache.kylin.job.streaming;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.cache.RemoteCacheUpdater;
@@ -42,6 +41,8 @@ import org.apache.kylin.common.restclient.AbstractRestCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
/**
*/
public class StreamingCLI {
@@ -60,26 +61,26 @@ public class StreamingCLI {
while (i < args.length) {
String argName = args[i];
switch (argName) {
- case "-oneoff":
- bootstrapConfig.setOneOff(Boolean.parseBoolean(args[++i]));
- break;
- case "-start":
- bootstrapConfig.setStart(Long.parseLong(args[++i]));
- break;
- case "-end":
- bootstrapConfig.setEnd(Long.parseLong(args[++i]));
- break;
- case "-streaming":
- bootstrapConfig.setStreaming(args[++i]);
- break;
- case "-partition":
- bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
- break;
- case "-margin":
- bootstrapConfig.setMargin(Long.parseLong(args[++i]));
- break;
- default:
- logger.warn("ignore this arg:" + argName);
+ case "-oneoff":
+ bootstrapConfig.setOneOff(Boolean.parseBoolean(args[++i]));
+ break;
+ case "-start":
+ bootstrapConfig.setStart(Long.parseLong(args[++i]));
+ break;
+ case "-end":
+ bootstrapConfig.setEnd(Long.parseLong(args[++i]));
+ break;
+ case "-streaming":
+ bootstrapConfig.setStreaming(args[++i]);
+ break;
+ case "-partition":
+ bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
+ break;
+ case "-margin":
+ bootstrapConfig.setMargin(Long.parseLong(args[++i]));
+ break;
+ default:
+ logger.warn("ignore this arg:" + argName);
}
i++;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
index cf5b8d1..3492f20 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
@@ -85,9 +85,14 @@ public class AggregateRegionObserver extends BaseRegionObserver {
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
- byte[] behavior = scan.getAttribute(BEHAVIOR);
- if (behavior != null && behavior.length != 0) {
- observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+ try {
+ byte[] behavior = scan.getAttribute(BEHAVIOR);
+ if (behavior != null && behavior.length != 0) {
+ observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+ }
+ } catch (Exception e) {
+ LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
+ observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
}
// start/end region operation & sync on scanner is suggested by the
@@ -102,7 +107,5 @@ public class AggregateRegionObserver extends BaseRegionObserver {
} finally {
region.closeRegionOperation();
}
-
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
new file mode 100644
index 0000000..dc18cc4
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
@@ -0,0 +1,83 @@
+package org.apache.kylin.streaming;
+
+import java.util.List;
+import java.util.concurrent.*;
+
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ */
+public class MarginCalculator implements Runnable {
+
+ private static final Logger logger = LoggerFactory.getLogger(MarginCalculator.class);
+
+
+ private final String streaming;
+ private final List<BlockingQueue<StreamMessage>> queues;
+ private final List<Pair<Long,Long>> firstAndLastOffsets;
+ private StreamParser streamParser;
+
+ public MarginCalculator(String streaming, List<BlockingQueue<StreamMessage>> queues, StreamParser streamParser, List<Pair<Long,Long>> firstAndLastOffsets) {
+ Preconditions.checkArgument(queues.size() > 0);
+ this.streaming = streaming;
+ this.queues = queues;
+ this.streamParser = streamParser;
+ this.firstAndLastOffsets = firstAndLastOffsets;
+ }
+
+ @Override
+ public void run() {
+ try {
+ final int inputCount = queues.size();
+ final ExecutorService executorService = Executors.newFixedThreadPool(inputCount);
+ final CountDownLatch countDownLatch = new CountDownLatch(inputCount);
+ final List<Future<MicroStreamBatch>> futures = Lists.newLinkedList();
+ int partitionId = 0;
+ for (BlockingQueue<StreamMessage> queue : queues) {
+ futures.add(executorService.submit(new StreamFetcher(partitionId, queue, countDownLatch, batchCondition, streamParser)));
+ }
+ countDownLatch.await();
+ List<MicroStreamBatch> batches = Lists.newLinkedList();
+ for (Future<MicroStreamBatch> future : futures) {
+ if (future.get() != null) {
+ batches.add(future.get());
+ } else {
+ logger.warn("EOF encountered, stop streaming");
+ }
+ }
+
+ MicroStreamBatch batch = batches.get(0);
+ if (batches.size() > 1) {
+ for (int i = 1; i < inputCount; i++) {
+ if (batches.get(i).size() > 0) {
+ batch = MicroStreamBatch.union(batch, batches.get(i));
+ }
+ }
+ }
+ batch.getTimestamp().setFirst(batchCondition.getStartTime());
+ batch.getTimestamp().setSecond(batchCondition.getEndTime());
+
+ logger.info("Consuming {} messages, covering from {} to {}", new String[] { String.valueOf(batch.size()), DateFormat.formatToTimeStr(batch.getTimestamp().getFirst()), DateFormat.formatToTimeStr(batch.getTimestamp().getSecond()) });
+ long startTime = System.currentTimeMillis();
+ consumer.consume(batch);
+ logger.info("Batch build costs {} milliseconds", System.currentTimeMillis() - startTime);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("this thread should not be interrupted", ie);
+ } catch (ExecutionException ee) {
+ logger.error("fetch stream error", ee);
+ throw new RuntimeException(ee);
+ } catch (Exception e) {
+ logger.error("error consume batch", e);
+ throw new RuntimeException("error consume batch", e);
+ }
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java b/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java
new file mode 100644
index 0000000..0e4cc3f
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java
@@ -0,0 +1,35 @@
+package org.apache.kylin.streaming;
+
+/**
+ */
+public class OffsetPeriodCondition implements BatchCondition {
+
+ private final long startOffset;
+ private final long endOffset;
+
+ public OffsetPeriodCondition(long startOffset, long endOffset) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ public long getStartOffset() {
+ return startOffset;
+ }
+
+ public long getEndOffset() {
+ return endOffset;
+ }
+
+ @Override
+ public Result apply(ParsedStreamMessage message) {
+ final long offset = message.getOffset();
+ if (offset < startOffset) {
+ return Result.DISCARD;
+ } else if (offset < endOffset) {
+ return Result.ACCEPT;
+ } else {
+ return Result.REJECT;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java
new file mode 100644
index 0000000..8f68a0a
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java
@@ -0,0 +1,13 @@
+package org.apache.kylin.streaming;
+
+/**
+ */
+public class PartitionMargin {
+ public long leftMargin;
+ public long rightMargin;
+
+ public PartitionMargin(long leftMargin, long rightMargin) {
+ this.leftMargin = leftMargin;
+ this.rightMargin = rightMargin;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java
new file mode 100644
index 0000000..a670ee4
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java
@@ -0,0 +1,103 @@
+package org.apache.kylin.streaming;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * get the margin for a single partition
+ */
+public class PartitionMarginCalculator implements Callable<PartitionMargin> {
+
+ /**
+ * use the WINDOW_SIZE messages prior to a message to represent its show time
+ */
+ private class MovingAverage {
+
+ private static final int WINDOW_SIZE = 20;
+
+ private Queue<Long> q = Lists.newLinkedList();
+ private long totalSum = 0;
+
+ public long addNewElementAndGetAvg(long e) {
+ if (q.size() < WINDOW_SIZE) {
+ q.add(e);
+ totalSum += e;
+ return totalSum / q.size();
+ }
+
+ long head = q.remove();
+ q.add(e);
+ return (totalSum - head + e) / WINDOW_SIZE;
+ }
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(PartitionMarginCalculator.class);
+
+ private final BlockingQueue<StreamMessage> streamMessageQueue;
+ private final CountDownLatch countDownLatch;
+ private final int partitionId;
+ private final long lastOffset;
+ private final StreamParser streamParser;
+
+ //runtime calculations
+ private final HashMap<Long, Long> earliestOffsets = Maps.newHashMap();//for each second in context, record its earliest show offset in kafka
+ private long maxMargin = 0;
+ private final MovingAverage average = new MovingAverage();
+
+ public PartitionMarginCalculator(int partitionId, BlockingQueue<StreamMessage> streamMessageQueue, CountDownLatch countDownLatch, long lastOffset, StreamParser streamParser) {
+ this.partitionId = partitionId;
+ this.streamMessageQueue = streamMessageQueue;
+ this.countDownLatch = countDownLatch;
+ this.streamParser = streamParser;
+ this.lastOffset = lastOffset;
+ }
+
+ @Override
+ public PartitionMargin call() throws Exception {
+ try {
+ while (true) {
+ StreamMessage streamMessage = streamMessageQueue.poll(30, TimeUnit.SECONDS);
+ if (streamMessage == null) {
+ logger.info("The stream queue for partition {} is drained", partitionId);
+ continue;
+ }
+
+ final ParsedStreamMessage parsedStreamMessage = streamParser.parse(streamMessage);
+ if (parsedStreamMessage == null) {
+ throw new RuntimeException("parsedStreamMessage of " + new String(streamMessage.getRawData()) + " is null");
+ }
+
+ if (parsedStreamMessage.getOffset() >= lastOffset) {
+ logger.info("The final max margin for partition {} is {} ", partitionId, maxMargin);
+ return new PartitionMargin(maxMargin, maxMargin);
+ }
+
+ long timestamp = parsedStreamMessage.getTimestamp();
+ long wallTime = average.addNewElementAndGetAvg(timestamp);
+ long formalizedTs = timestamp / 1000 * 1000;
+
+ if (earliestOffsets.containsKey(formalizedTs)) {
+ this.maxMargin = Math.max(this.maxMargin, Math.abs(earliestOffsets.get(formalizedTs) - wallTime));
+ } else {
+ earliestOffsets.put(formalizedTs, wallTime);
+ }
+
+ }
+ } catch (Exception e) {
+ logger.error("partition margin calculation stream error, stopping", e);
+ throw new RuntimeException("partition margin calculation stream error, stopping", e);
+ } finally {
+ logger.info("one partition sign off");
+ countDownLatch.countDown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
index afe6d79..38af7a1 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
@@ -10,7 +10,7 @@ import java.util.concurrent.CountDownLatch;
/**
*/
public class StreamFetcher implements Callable<MicroStreamBatch> {
- private static final Logger logger = LoggerFactory.getLogger(StreamBuilder.class);
+ private static final Logger logger = LoggerFactory.getLogger(StreamFetcher.class);
private final BlockingQueue<StreamMessage> streamMessageQueue;
private final CountDownLatch countDownLatch;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
index 160bdfe..264c384 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
@@ -7,6 +7,7 @@ import kafka.cluster.Broker;
import kafka.javaapi.FetchResponse;
import kafka.javaapi.PartitionMetadata;
import kafka.message.MessageAndOffset;
+import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,14 +57,21 @@ public final class StreamingUtil {
}
public static long findClosestOffsetWithDataTimestamp(KafkaClusterConfig kafkaClusterConfig, int partitionId, long timestamp, StreamParser streamParser) {
+ Pair<Long,Long> firstAndLast = getFirstAndLastOffset(kafkaClusterConfig, partitionId);
+ final String topic = kafkaClusterConfig.getTopic();
+
+ logger.info(String.format("topic: %s, partitionId: %d, try to find closest offset with timestamp: %d between offset {%d, %d}", topic, partitionId, timestamp, firstAndLast.getFirst(), firstAndLast.getSecond()));
+ final long result = binarySearch(kafkaClusterConfig, partitionId, firstAndLast.getFirst(), firstAndLast.getSecond(), timestamp, streamParser);
+ logger.info(String.format("topic: %s, partitionId: %d, found offset: %d", topic, partitionId, result));
+ return result;
+ }
+
+ public static Pair<Long, Long> getFirstAndLastOffset(KafkaClusterConfig kafkaClusterConfig, int partitionId) {
final String topic = kafkaClusterConfig.getTopic();
final Broker leadBroker = Preconditions.checkNotNull(getLeadBroker(kafkaClusterConfig, partitionId), "unable to find leadBroker with config:" + kafkaClusterConfig + " partitionId:" + partitionId);
final long earliestOffset = KafkaRequester.getLastOffset(topic, partitionId, OffsetRequest.EarliestTime(), leadBroker, kafkaClusterConfig);
final long latestOffset = KafkaRequester.getLastOffset(topic, partitionId, OffsetRequest.LatestTime(), leadBroker, kafkaClusterConfig) - 1;
- logger.info(String.format("topic: %s, partitionId: %d, try to find closest offset with timestamp: %d between offset {%d, %d}", topic, partitionId, timestamp, earliestOffset, latestOffset));
- final long result = binarySearch(kafkaClusterConfig, partitionId, earliestOffset, latestOffset, timestamp, streamParser);
- logger.info(String.format("topic: %s, partitionId: %d, found offset: %d", topic, partitionId, result));
- return result;
+ return Pair.newPair(earliestOffset, latestOffset);
}
private static long binarySearch(KafkaClusterConfig kafkaClusterConfig, int partitionId, long startOffset, long endOffset, long targetTimestamp, StreamParser streamParser) {
@@ -117,4 +125,12 @@ public final class StreamingUtil {
return parsedStreamMessage.getTimestamp();
}
+
+ public static void main(String[] args) {
+ if (args == null || args.length == 0) {
+ }
+
+ if ("calculatemargin".equals(args[0])) {
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
index cba52e8..908aa85 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
@@ -65,11 +65,6 @@ public final class TimedJsonStreamParser implements StreamParser {
private final ObjectMapper mapper = new ObjectMapper();
private final JavaType mapType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
- public TimedJsonStreamParser(List<TblColRef> allColumns) {
- this.allColumns = allColumns;
- this.formatTs = false;
- }
-
public TimedJsonStreamParser(List<TblColRef> allColumns, boolean formatTs) {
this.allColumns = allColumns;
this.formatTs = formatTs;
[3/4] incubator-kylin git commit: KYLIN-868 add a metadata
backup/restore script in bin folder
Posted by ma...@apache.org.
KYLIN-868 add a metadata backup/restore script in bin folder
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fa8aa126
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fa8aa126
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fa8aa126
Branch: refs/heads/0.8
Commit: fa8aa12663148d0578f5dec5bcc8498d494dd95f
Parents: a65c504
Author: honma <ho...@ebay.com>
Authored: Wed Jul 1 15:52:20 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800
----------------------------------------------------------------------
bin/metastore.sh | 37 +++++++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fa8aa126/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
new file mode 100644
index 0000000..f08de31
--- /dev/null
+++ b/bin/metastore.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
+_jobjar=`ls ${KYLIN_HOME}/lib |grep kylin-job`
+_fulljobjar="${KYLIN_HOME}/lib/${_jobjar}"
+
+if [ $1 == "backup" ]
+then
+
+ mkdir -p ${KYLIN_HOME}/meta_backups
+
+ _now=$(date +"%Y_%m_%d_%H_%M_%S")
+ _file="${KYLIN_HOME}/meta_backups/meta_${_now}"
+ echo "Starting backup to ${_file}"
+ mkdir -p ${_file}
+
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool copy ${KYLIN_HOME}/conf/kylin.properties ${_file}
+ echo "metadata store backed up to ${_file}"
+
+elif [ $1 == "restore" ]
+then
+
+ _file=$2
+ echo "Starting restoring $_file"
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool copy $_file ${KYLIN_HOME}/conf/kylin.properties
+
+elif [ $1 == "reset" ]
+then
+
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool reset
+
+else
+ echo "usage: metastore.sh backup or metastore.sh restore PATH_TO_LOCAL_META"
+ exit 1
+fi
[2/4] incubator-kylin git commit: KYLIN-865
Posted by ma...@apache.org.
KYLIN-865
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a65c5048
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a65c5048
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a65c5048
Branch: refs/heads/0.8
Commit: a65c50487abcad4834d3daf78a45ebbc0997ade3
Parents: 1657db8
Author: honma <ho...@ebay.com>
Authored: Tue Jun 30 17:24:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/AbstractKylinTestCase.java | 12 +++++++++---
.../org/apache/kylin/rest/service/BasicService.java | 4 ++--
.../org/apache/kylin/rest/service/CacheService.java | 8 ++++----
.../org/apache/kylin/rest/service/ServiceTestBase.java | 6 ++++--
4 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 3fd82b3..f196744 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -36,7 +36,15 @@ public abstract class AbstractKylinTestCase {
public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
- public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.metadata.project.ProjectManager", "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.metadata.realization.RealizationRegistry" };
+ public static final String[] SERVICES_WITH_CACHE = {//
+ "org.apache.kylin.cube.CubeManager",//
+ "org.apache.kylin.cube.CubeDescManager", //
+ "org.apache.kylin.invertedindex.IIDescManager",//
+ "org.apache.kylin.invertedindex.IIManager",//
+ "org.apache.kylin.metadata.realization.RealizationRegistry", //
+ "org.apache.kylin.metadata.project.ProjectManager", //
+ "org.apache.kylin.metadata.MetadataManager" //
+ };
public abstract void createTestMetadata() throws Exception;
@@ -46,8 +54,6 @@ public abstract class AbstractKylinTestCase {
return KylinConfig.getInstanceFromEnv();
}
-
-
public static void staticCleanupTestMetadata() {
cleanupCache();
System.clearProperty(KylinConfig.KYLIN_CONF);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index a025535..272007f 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -66,7 +66,7 @@ public abstract class BasicService {
private static ConcurrentMap<String, DataSource> olapDataSources = new ConcurrentHashMap<String, DataSource>();
@Autowired
- private CacheManager cacheManager;
+ private static CacheManager cacheManager;
// @Autowired
// protected JdbcTemplate jdbcTemplate;
@@ -81,7 +81,7 @@ public abstract class BasicService {
}
}
- protected void cleanAllDataCache() {
+ public static void cleanAllDataCache() {
if (cacheManager != null)
cacheManager.clearAll();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 6a301ae..57e469a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -27,7 +27,6 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
@@ -36,6 +35,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.rest.constant.Constant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,14 +105,14 @@ public class CacheService extends BasicService {
break;
case ALL:
MetadataManager.clearCache();
- DictionaryManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ RealizationRegistry.clearCache();
ProjectManager.clearCache();
- cleanAllDataCache();
- removeAllOLAPDataSources();
+ BasicService.cleanAllDataCache();
+ BasicService.removeAllOLAPDataSources();
break;
default:
throw new RuntimeException("invalid cacheType:" + cacheType);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 064badb..c2ba897 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -21,11 +21,11 @@ package org.apache.kylin.rest.service;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.junit.*;
import org.junit.runner.RunWith;
import org.springframework.security.authentication.TestingAuthenticationToken;
@@ -59,12 +59,14 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
this.createTestMetadata();
MetadataManager.clearCache();
- DictionaryManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ RealizationRegistry.clearCache();
ProjectManager.clearCache();
+ BasicService.cleanAllDataCache();
+ BasicService.removeAllOLAPDataSources();
}