You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/07/01 10:43:00 UTC

[1/4] incubator-kylin git commit: KYLIN-864 half way

Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8 3d83a253a -> 098f5800d


KYLIN-864 half way


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/098f5800
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/098f5800
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/098f5800

Branch: refs/heads/0.8
Commit: 098f5800db31ef75d00eb50f762bd78ecf73ca76
Parents: fa8aa12
Author: honma <ho...@ebay.com>
Authored: Wed Jul 1 16:42:15 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/streaming/StreamingBootstrap.java |  83 +++++-----
 .../kylin/streaming/MarginCalculator.java       | 166 +++++++++----------
 2 files changed, 122 insertions(+), 127 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/098f5800/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 864062f..d4c5139 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,24 +34,17 @@
 
 package org.apache.kylin.job.streaming;
 
-import java.lang.reflect.Constructor;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-import javax.annotation.Nullable;
-
-import jodd.io.StreamUtil;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import kafka.api.OffsetRequest;
 import kafka.cluster.Broker;
-
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.HBaseConnection;
 import org.apache.kylin.common.util.DaemonThreadFactory;
 import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.common.util.TimeUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -67,10 +60,12 @@ import org.apache.kylin.streaming.invertedindex.IIStreamConsumer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import javax.annotation.Nullable;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
 
 /**
  */
@@ -277,35 +272,35 @@ public class StreamingBootstrap {
         logger.info("one off build finished");
     }
 
-    private void startCalculatingMargin(final StreamingConfig streamingConfig) throws Exception {
-        final String cubeName = streamingConfig.getCubeName();
-        final StreamParser streamParser = getStreamParser(streamingConfig, Lists.<TblColRef>newArrayList());
-        final List<BlockingQueue<StreamMessage>> queues = Lists.newLinkedList();
-
-        int clusterId = 0;
-        final List<Pair<Long,Long>> firstAndLastOffsets = Lists.newArrayList();
-
-        for (final KafkaClusterConfig kafkaClusterConfig : streamingConfig.getKafkaClusterConfigs()) {
-            final ConcurrentMap<Integer, Long> partitionIdOffsetMap = Maps.newConcurrentMap();
-            final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
-            for (int i = 0; i < partitionCount; ++i) {
-                Pair<Long,Long> firstlast = StreamingUtil.getFirstAndLastOffset(kafkaClusterConfig,i);
-                firstAndLastOffsets.add(firstlast);
-                partitionIdOffsetMap.putIfAbsent(i,firstlast.getFirst());
-            }
-
-            logger.info("partitionId to start offset map:" + partitionIdOffsetMap);
-            Preconditions.checkArgument(partitionIdOffsetMap.size() == partitionCount, "fail to get all start offset");
-            final List<BlockingQueue<StreamMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, partitionIdOffsetMap, 0);
-            queues.addAll(oneClusterQueue);
-            logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
-            clusterId++;
-        }
-
-        OneOffStreamBuilder oneOffStreamBuilder = new OneOffStreamBuilder(streamingConfig.getName(), queues, streamParser, new CubeStreamConsumer(cubeName), startTimestamp, endTimestamp, margin);
-        Executors.newSingleThreadExecutor().submit(oneOffStreamBuilder).get();
-        logger.info("one off build finished");
-    }
+//    private void startCalculatingMargin(final StreamingConfig streamingConfig) throws Exception {
+//        final String cubeName = streamingConfig.getCubeName();
+//        final StreamParser streamParser = getStreamParser(streamingConfig, Lists.<TblColRef>newArrayList());
+//        final List<BlockingQueue<StreamMessage>> queues = Lists.newLinkedList();
+//
+//        int clusterId = 0;
+//        final List<Pair<Long,Long>> firstAndLastOffsets = Lists.newArrayList();
+//
+//        for (final KafkaClusterConfig kafkaClusterConfig : streamingConfig.getKafkaClusterConfigs()) {
+//            final ConcurrentMap<Integer, Long> partitionIdOffsetMap = Maps.newConcurrentMap();
+//            final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
+//            for (int i = 0; i < partitionCount; ++i) {
+//                Pair<Long,Long> firstlast = StreamingUtil.getFirstAndLastOffset(kafkaClusterConfig,i);
+//                firstAndLastOffsets.add(firstlast);
+//                partitionIdOffsetMap.putIfAbsent(i,firstlast.getFirst());
+//            }
+//
+//            logger.info("partitionId to start offset map:" + partitionIdOffsetMap);
+//            Preconditions.checkArgument(partitionIdOffsetMap.size() == partitionCount, "fail to get all start offset");
+//            final List<BlockingQueue<StreamMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, partitionIdOffsetMap, 0);
+//            queues.addAll(oneClusterQueue);
+//            logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
+//            clusterId++;
+//        }
+//
+//        OneOffStreamBuilder oneOffStreamBuilder = new OneOffStreamBuilder(streamingConfig.getName(), queues, streamParser, new CubeStreamConsumer(cubeName), startTimestamp, endTimestamp, margin);
+//        Executors.newSingleThreadExecutor().submit(oneOffStreamBuilder).get();
+//        logger.info("one off build finished");
+//    }
 
     private void startIIStreaming(StreamingConfig streamingConfig, final int partitionId) throws Exception {
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/098f5800/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
index dc18cc4..855a0f0 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
@@ -1,83 +1,83 @@
-package org.apache.kylin.streaming;
-
-import java.util.List;
-import java.util.concurrent.*;
-
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class MarginCalculator implements Runnable {
-
-    private static final Logger logger = LoggerFactory.getLogger(MarginCalculator.class);
-
-
-    private final String streaming;
-    private final List<BlockingQueue<StreamMessage>> queues;
-    private final List<Pair<Long,Long>> firstAndLastOffsets;
-    private StreamParser streamParser;
-
-    public MarginCalculator(String streaming, List<BlockingQueue<StreamMessage>> queues, StreamParser streamParser, List<Pair<Long,Long>> firstAndLastOffsets) {
-        Preconditions.checkArgument(queues.size() > 0);
-        this.streaming = streaming;
-        this.queues = queues;
-        this.streamParser = streamParser;
-        this.firstAndLastOffsets = firstAndLastOffsets;
-    }
-
-    @Override
-    public void run() {
-        try {
-            final int inputCount = queues.size();
-            final ExecutorService executorService = Executors.newFixedThreadPool(inputCount);
-            final CountDownLatch countDownLatch = new CountDownLatch(inputCount);
-            final List<Future<MicroStreamBatch>> futures = Lists.newLinkedList();
-            int partitionId = 0;
-            for (BlockingQueue<StreamMessage> queue : queues) {
-                futures.add(executorService.submit(new StreamFetcher(partitionId, queue, countDownLatch, batchCondition, streamParser)));
-            }
-            countDownLatch.await();
-            List<MicroStreamBatch> batches = Lists.newLinkedList();
-            for (Future<MicroStreamBatch> future : futures) {
-                if (future.get() != null) {
-                    batches.add(future.get());
-                } else {
-                    logger.warn("EOF encountered, stop streaming");
-                }
-            }
-
-            MicroStreamBatch batch = batches.get(0);
-            if (batches.size() > 1) {
-                for (int i = 1; i < inputCount; i++) {
-                    if (batches.get(i).size() > 0) {
-                        batch = MicroStreamBatch.union(batch, batches.get(i));
-                    }
-                }
-            }
-            batch.getTimestamp().setFirst(batchCondition.getStartTime());
-            batch.getTimestamp().setSecond(batchCondition.getEndTime());
-
-            logger.info("Consuming {} messages, covering from {} to {}", new String[] { String.valueOf(batch.size()), DateFormat.formatToTimeStr(batch.getTimestamp().getFirst()), DateFormat.formatToTimeStr(batch.getTimestamp().getSecond()) });
-            long startTime = System.currentTimeMillis();
-            consumer.consume(batch);
-            logger.info("Batch build costs {} milliseconds", System.currentTimeMillis() - startTime);
-        } catch (InterruptedException ie) {
-            throw new RuntimeException("this thread should not be interrupted", ie);
-        } catch (ExecutionException ee) {
-            logger.error("fetch stream error", ee);
-            throw new RuntimeException(ee);
-        } catch (Exception e) {
-            logger.error("error consume batch", e);
-            throw new RuntimeException("error consume batch", e);
-        }
-
-
-    }
-
-}
+//package org.apache.kylin.streaming;
+//
+//import java.util.List;
+//import java.util.concurrent.*;
+//
+//import org.apache.kylin.common.util.DateFormat;
+//import org.apache.kylin.common.util.Pair;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//
+//import com.google.common.base.Preconditions;
+//import com.google.common.collect.Lists;
+//
+///**
+// */
+//public class MarginCalculator implements Runnable {
+//
+//    private static final Logger logger = LoggerFactory.getLogger(MarginCalculator.class);
+//
+//
+//    private final String streaming;
+//    private final List<BlockingQueue<StreamMessage>> queues;
+//    private final List<Pair<Long,Long>> firstAndLastOffsets;
+//    private StreamParser streamParser;
+//
+//    public MarginCalculator(String streaming, List<BlockingQueue<StreamMessage>> queues, StreamParser streamParser, List<Pair<Long,Long>> firstAndLastOffsets) {
+//        Preconditions.checkArgument(queues.size() > 0);
+//        this.streaming = streaming;
+//        this.queues = queues;
+//        this.streamParser = streamParser;
+//        this.firstAndLastOffsets = firstAndLastOffsets;
+//    }
+//
+//    @Override
+//    public void run() {
+//        try {
+//            final int inputCount = queues.size();
+//            final ExecutorService executorService = Executors.newFixedThreadPool(inputCount);
+//            final CountDownLatch countDownLatch = new CountDownLatch(inputCount);
+//            final List<Future<MicroStreamBatch>> futures = Lists.newLinkedList();
+//            int partitionId = 0;
+//            for (BlockingQueue<StreamMessage> queue : queues) {
+//                futures.add(executorService.submit(new StreamFetcher(partitionId, queue, countDownLatch, batchCondition, streamParser)));
+//            }
+//            countDownLatch.await();
+//            List<MicroStreamBatch> batches = Lists.newLinkedList();
+//            for (Future<MicroStreamBatch> future : futures) {
+//                if (future.get() != null) {
+//                    batches.add(future.get());
+//                } else {
+//                    logger.warn("EOF encountered, stop streaming");
+//                }
+//            }
+//
+//            MicroStreamBatch batch = batches.get(0);
+//            if (batches.size() > 1) {
+//                for (int i = 1; i < inputCount; i++) {
+//                    if (batches.get(i).size() > 0) {
+//                        batch = MicroStreamBatch.union(batch, batches.get(i));
+//                    }
+//                }
+//            }
+//            batch.getTimestamp().setFirst(batchCondition.getStartTime());
+//            batch.getTimestamp().setSecond(batchCondition.getEndTime());
+//
+//            logger.info("Consuming {} messages, covering from {} to {}", new String[] { String.valueOf(batch.size()), DateFormat.formatToTimeStr(batch.getTimestamp().getFirst()), DateFormat.formatToTimeStr(batch.getTimestamp().getSecond()) });
+//            long startTime = System.currentTimeMillis();
+//            consumer.consume(batch);
+//            logger.info("Batch build costs {} milliseconds", System.currentTimeMillis() - startTime);
+//        } catch (InterruptedException ie) {
+//            throw new RuntimeException("this thread should not be interrupted", ie);
+//        } catch (ExecutionException ee) {
+//            logger.error("fetch stream error", ee);
+//            throw new RuntimeException(ee);
+//        } catch (Exception e) {
+//            logger.error("error consume batch", e);
+//            throw new RuntimeException("error consume batch", e);
+//        }
+//
+//
+//    }
+//
+//}


[4/4] incubator-kylin git commit: KYLIN-864

Posted by ma...@apache.org.
KYLIN-864


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1657db8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1657db8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1657db8f

Branch: refs/heads/0.8
Commit: 1657db8f50c641496fabc8d4424ed9b62fd16d33
Parents: 3d83a25
Author: honma <ho...@ebay.com>
Authored: Tue Jun 30 16:30:49 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800

----------------------------------------------------------------------
 .../kylin/common/debug/BackdoorToggles.java     |  12 +--
 .../kylin/job/streaming/BootstrapConfig.java    |   1 +
 .../kylin/job/streaming/StreamingBootstrap.java |  55 ++++++++--
 .../kylin/job/streaming/StreamingCLI.java       |  43 ++++----
 .../observer/AggregateRegionObserver.java       |  13 ++-
 .../kylin/streaming/MarginCalculator.java       |  83 +++++++++++++++
 .../kylin/streaming/OffsetPeriodCondition.java  |  35 +++++++
 .../apache/kylin/streaming/PartitionMargin.java |  13 +++
 .../streaming/PartitionMarginCalculator.java    | 103 +++++++++++++++++++
 .../apache/kylin/streaming/StreamFetcher.java   |   2 +-
 .../apache/kylin/streaming/StreamingUtil.java   |  24 ++++-
 .../kylin/streaming/TimedJsonStreamParser.java  |   5 -
 12 files changed, 337 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 5ce76aa..560a836 100644
--- a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -64,9 +64,9 @@ public class BackdoorToggles {
      *
      *
      *
-     example:
+     example:(put it into request body)
      "backdoorToggles": {
-     "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+        "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
      }
      */
     public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
@@ -74,9 +74,9 @@ public class BackdoorToggles {
     /**
      * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
      *
-     example:
+     example:(put it into request body)
      "backdoorToggles": {
-     "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+        "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
      }
      */
     public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
@@ -84,9 +84,9 @@ public class BackdoorToggles {
     /**
      * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
      *
-     example:
+     example:(put it into request body)
      "backdoorToggles": {
-     "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
+        "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
      }
      */
     public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
index a82fec3..302f455 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
@@ -12,6 +12,7 @@ public class BootstrapConfig {
     private long end = 0L;
     private long margin = 0L;
 
+
     public long getMargin() {
         return margin;
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 98bbcd4..864062f 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,17 +34,24 @@
 
 package org.apache.kylin.job.streaming;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+import javax.annotation.Nullable;
+
+import jodd.io.StreamUtil;
 import kafka.api.OffsetRequest;
 import kafka.cluster.Broker;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.HBaseConnection;
 import org.apache.kylin.common.util.DaemonThreadFactory;
 import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.common.util.TimeUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -60,12 +67,10 @@ import org.apache.kylin.streaming.invertedindex.IIStreamConsumer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
-import java.lang.reflect.Constructor;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 /**
  */
@@ -272,6 +277,36 @@ public class StreamingBootstrap {
         logger.info("one off build finished");
     }
 
+    private void startCalculatingMargin(final StreamingConfig streamingConfig) throws Exception {
+        final String cubeName = streamingConfig.getCubeName();
+        final StreamParser streamParser = getStreamParser(streamingConfig, Lists.<TblColRef>newArrayList());
+        final List<BlockingQueue<StreamMessage>> queues = Lists.newLinkedList();
+
+        int clusterId = 0;
+        final List<Pair<Long,Long>> firstAndLastOffsets = Lists.newArrayList();
+
+        for (final KafkaClusterConfig kafkaClusterConfig : streamingConfig.getKafkaClusterConfigs()) {
+            final ConcurrentMap<Integer, Long> partitionIdOffsetMap = Maps.newConcurrentMap();
+            final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
+            for (int i = 0; i < partitionCount; ++i) {
+                Pair<Long,Long> firstlast = StreamingUtil.getFirstAndLastOffset(kafkaClusterConfig,i);
+                firstAndLastOffsets.add(firstlast);
+                partitionIdOffsetMap.putIfAbsent(i,firstlast.getFirst());
+            }
+
+            logger.info("partitionId to start offset map:" + partitionIdOffsetMap);
+            Preconditions.checkArgument(partitionIdOffsetMap.size() == partitionCount, "fail to get all start offset");
+            final List<BlockingQueue<StreamMessage>> oneClusterQueue = consume(clusterId, kafkaClusterConfig, partitionCount, partitionIdOffsetMap, 0);
+            queues.addAll(oneClusterQueue);
+            logger.info("Cluster {} with {} partitions", clusterId, oneClusterQueue.size());
+            clusterId++;
+        }
+
+        OneOffStreamBuilder oneOffStreamBuilder = new OneOffStreamBuilder(streamingConfig.getName(), queues, streamParser, new CubeStreamConsumer(cubeName), startTimestamp, endTimestamp, margin);
+        Executors.newSingleThreadExecutor().submit(oneOffStreamBuilder).get();
+        logger.info("one off build finished");
+    }
+
     private void startIIStreaming(StreamingConfig streamingConfig, final int partitionId) throws Exception {
 
         List<KafkaClusterConfig> allClustersConfigs = streamingConfig.getKafkaClusterConfigs();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index 409c558..1eb23f1 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -34,7 +34,6 @@
 
 package org.apache.kylin.job.streaming;
 
-import com.google.common.base.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.cache.RemoteCacheUpdater;
@@ -42,6 +41,8 @@ import org.apache.kylin.common.restclient.AbstractRestCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
+
 /**
  */
 public class StreamingCLI {
@@ -60,26 +61,26 @@ public class StreamingCLI {
             while (i < args.length) {
                 String argName = args[i];
                 switch (argName) {
-                    case "-oneoff":
-                        bootstrapConfig.setOneOff(Boolean.parseBoolean(args[++i]));
-                        break;
-                    case "-start":
-                        bootstrapConfig.setStart(Long.parseLong(args[++i]));
-                        break;
-                    case "-end":
-                        bootstrapConfig.setEnd(Long.parseLong(args[++i]));
-                        break;
-                    case "-streaming":
-                        bootstrapConfig.setStreaming(args[++i]);
-                        break;
-                    case "-partition":
-                        bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
-                        break;
-                    case "-margin":
-                        bootstrapConfig.setMargin(Long.parseLong(args[++i]));
-                        break;
-                    default:
-                        logger.warn("ignore this arg:" + argName);
+                case "-oneoff":
+                    bootstrapConfig.setOneOff(Boolean.parseBoolean(args[++i]));
+                    break;
+                case "-start":
+                    bootstrapConfig.setStart(Long.parseLong(args[++i]));
+                    break;
+                case "-end":
+                    bootstrapConfig.setEnd(Long.parseLong(args[++i]));
+                    break;
+                case "-streaming":
+                    bootstrapConfig.setStreaming(args[++i]);
+                    break;
+                case "-partition":
+                    bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
+                    break;
+                case "-margin":
+                    bootstrapConfig.setMargin(Long.parseLong(args[++i]));
+                    break;
+                default:
+                    logger.warn("ignore this arg:" + argName);
                 }
                 i++;
             }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
index cf5b8d1..3492f20 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
@@ -85,9 +85,14 @@ public class AggregateRegionObserver extends BaseRegionObserver {
         CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
 
         ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
-        byte[] behavior = scan.getAttribute(BEHAVIOR);
-        if (behavior != null && behavior.length != 0) {
-            observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+        try {
+            byte[] behavior = scan.getAttribute(BEHAVIOR);
+            if (behavior != null && behavior.length != 0) {
+                observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+            }
+        } catch (Exception e) {
+            LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
+            observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
         }
 
         // start/end region operation & sync on scanner is suggested by the
@@ -102,7 +107,5 @@ public class AggregateRegionObserver extends BaseRegionObserver {
         } finally {
             region.closeRegionOperation();
         }
-
     }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
new file mode 100644
index 0000000..dc18cc4
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/MarginCalculator.java
@@ -0,0 +1,83 @@
+package org.apache.kylin.streaming;
+
+import java.util.List;
+import java.util.concurrent.*;
+
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ */
+public class MarginCalculator implements Runnable {
+
+    private static final Logger logger = LoggerFactory.getLogger(MarginCalculator.class);
+
+
+    private final String streaming;
+    private final List<BlockingQueue<StreamMessage>> queues;
+    private final List<Pair<Long,Long>> firstAndLastOffsets;
+    private StreamParser streamParser;
+
+    public MarginCalculator(String streaming, List<BlockingQueue<StreamMessage>> queues, StreamParser streamParser, List<Pair<Long,Long>> firstAndLastOffsets) {
+        Preconditions.checkArgument(queues.size() > 0);
+        this.streaming = streaming;
+        this.queues = queues;
+        this.streamParser = streamParser;
+        this.firstAndLastOffsets = firstAndLastOffsets;
+    }
+
+    @Override
+    public void run() {
+        try {
+            final int inputCount = queues.size();
+            final ExecutorService executorService = Executors.newFixedThreadPool(inputCount);
+            final CountDownLatch countDownLatch = new CountDownLatch(inputCount);
+            final List<Future<MicroStreamBatch>> futures = Lists.newLinkedList();
+            int partitionId = 0;
+            for (BlockingQueue<StreamMessage> queue : queues) {
+                futures.add(executorService.submit(new StreamFetcher(partitionId, queue, countDownLatch, batchCondition, streamParser)));
+            }
+            countDownLatch.await();
+            List<MicroStreamBatch> batches = Lists.newLinkedList();
+            for (Future<MicroStreamBatch> future : futures) {
+                if (future.get() != null) {
+                    batches.add(future.get());
+                } else {
+                    logger.warn("EOF encountered, stop streaming");
+                }
+            }
+
+            MicroStreamBatch batch = batches.get(0);
+            if (batches.size() > 1) {
+                for (int i = 1; i < inputCount; i++) {
+                    if (batches.get(i).size() > 0) {
+                        batch = MicroStreamBatch.union(batch, batches.get(i));
+                    }
+                }
+            }
+            batch.getTimestamp().setFirst(batchCondition.getStartTime());
+            batch.getTimestamp().setSecond(batchCondition.getEndTime());
+
+            logger.info("Consuming {} messages, covering from {} to {}", new String[] { String.valueOf(batch.size()), DateFormat.formatToTimeStr(batch.getTimestamp().getFirst()), DateFormat.formatToTimeStr(batch.getTimestamp().getSecond()) });
+            long startTime = System.currentTimeMillis();
+            consumer.consume(batch);
+            logger.info("Batch build costs {} milliseconds", System.currentTimeMillis() - startTime);
+        } catch (InterruptedException ie) {
+            throw new RuntimeException("this thread should not be interrupted", ie);
+        } catch (ExecutionException ee) {
+            logger.error("fetch stream error", ee);
+            throw new RuntimeException(ee);
+        } catch (Exception e) {
+            logger.error("error consume batch", e);
+            throw new RuntimeException("error consume batch", e);
+        }
+
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java b/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java
new file mode 100644
index 0000000..0e4cc3f
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/OffsetPeriodCondition.java
@@ -0,0 +1,35 @@
+package org.apache.kylin.streaming;
+
+/**
+ */
+public class OffsetPeriodCondition implements BatchCondition {
+
+    private final long startOffset;
+    private final long endOffset;
+
+    public OffsetPeriodCondition(long startOffset, long endOffset) {
+        this.startOffset = startOffset;
+        this.endOffset = endOffset;
+    }
+
+    public long getStartOffset() {
+        return startOffset;
+    }
+
+    public long getEndOffset() {
+        return endOffset;
+    }
+
+    @Override
+    public Result apply(ParsedStreamMessage message) {
+        final long offset = message.getOffset();
+        if (offset < startOffset) {
+            return Result.DISCARD;
+        } else if (offset < endOffset) {
+            return Result.ACCEPT;
+        } else {
+            return Result.REJECT;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java
new file mode 100644
index 0000000..8f68a0a
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMargin.java
@@ -0,0 +1,13 @@
+package org.apache.kylin.streaming;
+
+/**
+ */
+public class PartitionMargin {
+    public long leftMargin;
+    public long rightMargin;
+
+    public PartitionMargin(long leftMargin, long rightMargin) {
+        this.leftMargin = leftMargin;
+        this.rightMargin = rightMargin;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java
new file mode 100644
index 0000000..a670ee4
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/PartitionMarginCalculator.java
@@ -0,0 +1,103 @@
+package org.apache.kylin.streaming;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * get the margin for a single partition
+ */
+public class PartitionMarginCalculator implements Callable<PartitionMargin> {
+
+    /**
+     * use the WINDOW_SIZE messages prior to a message to represent its show time
+     */
+    private class MovingAverage {
+
+        private static final int WINDOW_SIZE = 20;
+
+        private Queue<Long> q = Lists.newLinkedList();
+        private long totalSum = 0;
+
+        public long addNewElementAndGetAvg(long e) {
+            if (q.size() < WINDOW_SIZE) {
+                q.add(e);
+                totalSum += e;
+                return totalSum / q.size();
+            }
+
+            long head = q.remove();
+            q.add(e);
+            return (totalSum - head + e) / WINDOW_SIZE;
+        }
+    }
+
+    private static final Logger logger = LoggerFactory.getLogger(PartitionMarginCalculator.class);
+
+    private final BlockingQueue<StreamMessage> streamMessageQueue;
+    private final CountDownLatch countDownLatch;
+    private final int partitionId;
+    private final long lastOffset;
+    private final StreamParser streamParser;
+
+    //runtime calculations
+    private final HashMap<Long, Long> earliestOffsets = Maps.newHashMap();//for each second in context, record its earliest show offset in kafka
+    private long maxMargin = 0;
+    private final MovingAverage average = new MovingAverage();
+
+    public PartitionMarginCalculator(int partitionId, BlockingQueue<StreamMessage> streamMessageQueue, CountDownLatch countDownLatch, long lastOffset, StreamParser streamParser) {
+        this.partitionId = partitionId;
+        this.streamMessageQueue = streamMessageQueue;
+        this.countDownLatch = countDownLatch;
+        this.streamParser = streamParser;
+        this.lastOffset = lastOffset;
+    }
+
+    @Override
+    public PartitionMargin call() throws Exception {
+        try {
+            while (true) {
+                StreamMessage streamMessage = streamMessageQueue.poll(30, TimeUnit.SECONDS);
+                if (streamMessage == null) {
+                    logger.info("The stream queue for partition {} is drained", partitionId);
+                    continue;
+                }
+
+                final ParsedStreamMessage parsedStreamMessage = streamParser.parse(streamMessage);
+                if (parsedStreamMessage == null) {
+                    throw new RuntimeException("parsedStreamMessage of " + new String(streamMessage.getRawData()) + " is null");
+                }
+
+                if (parsedStreamMessage.getOffset() >= lastOffset) {
+                    logger.info("The final max margin for partition {} is {} ", partitionId, maxMargin);
+                    return new PartitionMargin(maxMargin, maxMargin);
+                }
+
+                long timestamp = parsedStreamMessage.getTimestamp();
+                long wallTime = average.addNewElementAndGetAvg(timestamp);
+                long formalizedTs = timestamp / 1000 * 1000;
+
+                if (earliestOffsets.containsKey(formalizedTs)) {
+                    this.maxMargin = Math.max(this.maxMargin, Math.abs(earliestOffsets.get(formalizedTs) - wallTime));
+                } else {
+                    earliestOffsets.put(formalizedTs, wallTime);
+                }
+
+            }
+        } catch (Exception e) {
+            logger.error("partition margin calculation stream error, stopping", e);
+            throw new RuntimeException("partition margin calculation stream error, stopping", e);
+        } finally {
+            logger.info("one partition sign off");
+            countDownLatch.countDown();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
index afe6d79..38af7a1 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
@@ -10,7 +10,7 @@ import java.util.concurrent.CountDownLatch;
 /**
  */
 public class StreamFetcher implements Callable<MicroStreamBatch> {
-    private static final Logger logger = LoggerFactory.getLogger(StreamBuilder.class);
+    private static final Logger logger = LoggerFactory.getLogger(StreamFetcher.class);
 
     private final BlockingQueue<StreamMessage> streamMessageQueue;
     private final CountDownLatch countDownLatch;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
index 160bdfe..264c384 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingUtil.java
@@ -7,6 +7,7 @@ import kafka.cluster.Broker;
 import kafka.javaapi.FetchResponse;
 import kafka.javaapi.PartitionMetadata;
 import kafka.message.MessageAndOffset;
+import org.apache.kylin.common.util.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,14 +57,21 @@ public final class StreamingUtil {
     }
 
     public static long findClosestOffsetWithDataTimestamp(KafkaClusterConfig kafkaClusterConfig, int partitionId, long timestamp, StreamParser streamParser) {
+        Pair<Long,Long> firstAndLast = getFirstAndLastOffset(kafkaClusterConfig, partitionId);
+        final String topic = kafkaClusterConfig.getTopic();
+
+        logger.info(String.format("topic: %s, partitionId: %d, try to find closest offset with timestamp: %d between offset {%d, %d}", topic, partitionId, timestamp, firstAndLast.getFirst(), firstAndLast.getSecond()));
+        final long result = binarySearch(kafkaClusterConfig, partitionId, firstAndLast.getFirst(), firstAndLast.getSecond(), timestamp, streamParser);
+        logger.info(String.format("topic: %s, partitionId: %d, found offset: %d", topic, partitionId, result));
+        return result;
+    }
+
+    public static Pair<Long, Long> getFirstAndLastOffset(KafkaClusterConfig kafkaClusterConfig, int partitionId) {
         final String topic = kafkaClusterConfig.getTopic();
         final Broker leadBroker = Preconditions.checkNotNull(getLeadBroker(kafkaClusterConfig, partitionId), "unable to find leadBroker with config:" + kafkaClusterConfig + " partitionId:" + partitionId);
         final long earliestOffset = KafkaRequester.getLastOffset(topic, partitionId, OffsetRequest.EarliestTime(), leadBroker, kafkaClusterConfig);
         final long latestOffset = KafkaRequester.getLastOffset(topic, partitionId, OffsetRequest.LatestTime(), leadBroker, kafkaClusterConfig) - 1;
-        logger.info(String.format("topic: %s, partitionId: %d, try to find closest offset with timestamp: %d between offset {%d, %d}", topic, partitionId, timestamp, earliestOffset, latestOffset));
-        final long result = binarySearch(kafkaClusterConfig, partitionId, earliestOffset, latestOffset, timestamp, streamParser);
-        logger.info(String.format("topic: %s, partitionId: %d, found offset: %d", topic, partitionId, result));
-        return result;
+        return Pair.newPair(earliestOffset, latestOffset);
     }
 
     private static long binarySearch(KafkaClusterConfig kafkaClusterConfig, int partitionId, long startOffset, long endOffset, long targetTimestamp, StreamParser streamParser) {
@@ -117,4 +125,12 @@ public final class StreamingUtil {
         return parsedStreamMessage.getTimestamp();
 
     }
+
+    public static void main(String[] args) {
+        if (args == null || args.length == 0) {
+        }
+
+        if ("calculatemargin".equals(args[0])) {
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1657db8f/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
index cba52e8..908aa85 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
@@ -65,11 +65,6 @@ public final class TimedJsonStreamParser implements StreamParser {
     private final ObjectMapper mapper = new ObjectMapper();
     private final JavaType mapType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
 
-    public TimedJsonStreamParser(List<TblColRef> allColumns) {
-        this.allColumns = allColumns;
-        this.formatTs = false;
-    }
-
     public TimedJsonStreamParser(List<TblColRef> allColumns, boolean formatTs) {
         this.allColumns = allColumns;
         this.formatTs = formatTs;


[3/4] incubator-kylin git commit: KYLIN-868 add a metadata backup/restore script in bin folder

Posted by ma...@apache.org.
KYLIN-868 add a metadata backup/restore script in bin folder


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fa8aa126
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fa8aa126
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fa8aa126

Branch: refs/heads/0.8
Commit: fa8aa12663148d0578f5dec5bcc8498d494dd95f
Parents: a65c504
Author: honma <ho...@ebay.com>
Authored: Wed Jul 1 15:52:20 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800

----------------------------------------------------------------------
 bin/metastore.sh | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fa8aa126/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
new file mode 100644
index 0000000..f08de31
--- /dev/null
+++ b/bin/metastore.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
+_jobjar=`ls ${KYLIN_HOME}/lib |grep kylin-job`
+_fulljobjar="${KYLIN_HOME}/lib/${_jobjar}"
+
+if [ $1 == "backup" ]
+then
+
+    mkdir -p ${KYLIN_HOME}/meta_backups
+
+    _now=$(date +"%Y_%m_%d_%H_%M_%S")
+    _file="${KYLIN_HOME}/meta_backups/meta_${_now}"
+    echo "Starting backup to ${_file}"
+    mkdir -p ${_file}
+
+    hbase  org.apache.hadoop.util.RunJar ${_fulljobjar}   org.apache.kylin.common.persistence.ResourceTool  copy ${KYLIN_HOME}/conf/kylin.properties ${_file}
+    echo "metadata store backed up to ${_file}"
+
+elif [ $1 == "restore" ]
+then
+
+    _file=$2
+    echo "Starting restoring $_file"
+    hbase  org.apache.hadoop.util.RunJar  ${_fulljobjar}   org.apache.kylin.common.persistence.ResourceTool  copy $_file ${KYLIN_HOME}/conf/kylin.properties
+
+elif [ $1 == "reset" ]
+then
+
+    hbase  org.apache.hadoop.util.RunJar ${_fulljobjar}   org.apache.kylin.common.persistence.ResourceTool  reset
+
+else
+    echo "usage: metastore.sh backup or metastore.sh restore PATH_TO_LOCAL_META"
+    exit 1
+fi


[2/4] incubator-kylin git commit: KYLIN-865

Posted by ma...@apache.org.
KYLIN-865


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a65c5048
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a65c5048
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a65c5048

Branch: refs/heads/0.8
Commit: a65c50487abcad4834d3daf78a45ebbc0997ade3
Parents: 1657db8
Author: honma <ho...@ebay.com>
Authored: Tue Jun 30 17:24:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 16:42:29 2015 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/util/AbstractKylinTestCase.java | 12 +++++++++---
 .../org/apache/kylin/rest/service/BasicService.java     |  4 ++--
 .../org/apache/kylin/rest/service/CacheService.java     |  8 ++++----
 .../org/apache/kylin/rest/service/ServiceTestBase.java  |  6 ++++--
 4 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 3fd82b3..f196744 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -36,7 +36,15 @@ public abstract class AbstractKylinTestCase {
 
     public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
 
-    public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.metadata.project.ProjectManager", "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.metadata.realization.RealizationRegistry" };
+    public static final String[] SERVICES_WITH_CACHE = {//
+    "org.apache.kylin.cube.CubeManager",//
+            "org.apache.kylin.cube.CubeDescManager", //
+            "org.apache.kylin.invertedindex.IIDescManager",//
+            "org.apache.kylin.invertedindex.IIManager",//
+            "org.apache.kylin.metadata.realization.RealizationRegistry", //
+            "org.apache.kylin.metadata.project.ProjectManager", //
+            "org.apache.kylin.metadata.MetadataManager" //
+    };
 
     public abstract void createTestMetadata() throws Exception;
 
@@ -46,8 +54,6 @@ public abstract class AbstractKylinTestCase {
         return KylinConfig.getInstanceFromEnv();
     }
 
-
-
     public static void staticCleanupTestMetadata() {
         cleanupCache();
         System.clearProperty(KylinConfig.KYLIN_CONF);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index a025535..272007f 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -66,7 +66,7 @@ public abstract class BasicService {
     private static ConcurrentMap<String, DataSource> olapDataSources = new ConcurrentHashMap<String, DataSource>();
 
     @Autowired
-    private CacheManager cacheManager;
+    private static CacheManager cacheManager;
 
     //    @Autowired
     //    protected JdbcTemplate jdbcTemplate;
@@ -81,7 +81,7 @@ public abstract class BasicService {
         }
     }
 
-    protected void cleanAllDataCache() {
+    public static void cleanAllDataCache() {
         if (cacheManager != null)
             cacheManager.clearAll();
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 6a301ae..57e469a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -27,7 +27,6 @@ import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.invertedindex.IIDescManager;
 import org.apache.kylin.invertedindex.IIManager;
 import org.apache.kylin.job.cube.CubingJob;
@@ -36,6 +35,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
 import org.apache.kylin.rest.constant.Constant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -105,14 +105,14 @@ public class CacheService extends BasicService {
                 break;
             case ALL:
                 MetadataManager.clearCache();
-                DictionaryManager.clearCache();
                 CubeDescManager.clearCache();
                 CubeManager.clearCache();
                 IIDescManager.clearCache();
                 IIManager.clearCache();
+                RealizationRegistry.clearCache();
                 ProjectManager.clearCache();
-                cleanAllDataCache();
-                removeAllOLAPDataSources();
+                BasicService.cleanAllDataCache();
+                BasicService.removeAllOLAPDataSources();
                 break;
             default:
                 throw new RuntimeException("invalid cacheType:" + cacheType);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a65c5048/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 064badb..c2ba897 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -21,11 +21,11 @@ package org.apache.kylin.rest.service;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.invertedindex.IIDescManager;
 import org.apache.kylin.invertedindex.IIManager;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
 import org.junit.*;
 import org.junit.runner.RunWith;
 import org.springframework.security.authentication.TestingAuthenticationToken;
@@ -59,12 +59,14 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
         this.createTestMetadata();
 
         MetadataManager.clearCache();
-        DictionaryManager.clearCache();
         CubeDescManager.clearCache();
         CubeManager.clearCache();
         IIDescManager.clearCache();
         IIManager.clearCache();
+        RealizationRegistry.clearCache();
         ProjectManager.clearCache();
+        BasicService.cleanAllDataCache();
+        BasicService.removeAllOLAPDataSources();
 
     }