You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/24 15:23:28 UTC
[1/4] incubator-kylin git commit: KYLIN-844 control observer behavior
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging 198be9e06 -> 269d92253
KYLIN-844 control observer behavior
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ad4a28af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ad4a28af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ad4a28af
Branch: refs/heads/0.7-staging
Commit: ad4a28af5e07fb75ec5d9a5eda11cd7409b033b1
Parents: 198be9e
Author: honma <ho...@ebay.com>
Authored: Fri Jun 19 18:29:59 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 12:02:25 2015 +0800
----------------------------------------------------------------------
.../observer/AggregateRegionObserver.java | 17 +++++---
.../observer/AggregationScanner.java | 37 +++++++++-------
.../coprocessor/observer/ObserverBehavior.java | 9 ++++
.../coprocessor/observer/ObserverEnabler.java | 33 +++++++--------
.../observer/AggregateRegionObserverTest.java | 44 ++++++++++----------
5 files changed, 80 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
index d893621..aea2cea 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
@@ -18,11 +18,6 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.Scan;
@@ -32,6 +27,11 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+
+import java.io.IOException;
/**
* @author yangli9
@@ -46,6 +46,8 @@ public class AggregateRegionObserver extends BaseRegionObserver {
static final String PROJECTOR = "_Projector";
static final String AGGREGATORS = "_Aggregators";
static final String FILTER = "_Filter";
+ static final String BEHAVIOR = "_Behavior";
+
@Override
public final RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException {
@@ -83,6 +85,9 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
+ byte[] behavior = scan.getAttribute(BEHAVIOR);
+ ObserverBehavior observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
@@ -90,7 +95,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner,observerBehavior);
}
} finally {
region.closeRegionOperation();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
index 6bdb53a..85f3f6e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
@@ -18,18 +18,17 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-
import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
/**
* @author yangli9
@@ -38,11 +37,14 @@ import org.apache.kylin.metadata.measure.MeasureAggregator;
public class AggregationScanner implements RegionScanner {
private RegionScanner outerScanner;
+ private ObserverBehavior behavior;
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner) throws IOException {
+ public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, ObserverBehavior behavior) throws IOException {
AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
+ this.behavior = behavior;
+
ObserverAggregationCache aggCache;
Stats stats = new Stats();
@@ -72,14 +74,19 @@ public class AggregationScanner implements RegionScanner {
Cell cell = results.get(0);
tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- if (filter != null && filter.evaluate(tuple) == false)
- continue;
- CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(results);
- MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
- aggregators.aggregate(bufs, results);
+ if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER.ordinal()) {
+ if (filter != null && filter.evaluate(tuple) == false)
+ continue;
- aggCache.checkMemoryUsage();
+ if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR.ordinal()) {
+ CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(results);
+ MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
+ aggregators.aggregate(bufs, results);
+
+ aggCache.checkMemoryUsage();
+ }
+ }
}
return aggCache;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
new file mode 100644
index 0000000..62c5bac
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
@@ -0,0 +1,9 @@
+package org.apache.kylin.storage.hbase.coprocessor.observer;
+
+/**
+ */
+public enum ObserverBehavior {
+ SCAN, //only scan data, used for profiling tuple scan speed
+ SCAN_FILTER, //only scan+filter used,used for profiling filter speed
+ SCAN_FILTER_AGGR, //default normal behavior
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index eff0eee..ce9ba9d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -18,32 +18,31 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.storage.StorageContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowValueDecoder;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.RegionScannerAdapter;
import org.apache.kylin.storage.hbase.ResultScannerAdapter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
/**
* @author yangli9
@@ -70,10 +69,11 @@ public class ObserverEnabler {
if (DEBUG_LOCAL_COPROCESSOR) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 });
+ scan.setAttribute(AggregateRegionObserver.BEHAVIOR, ObserverBehavior.SCAN_FILTER_AGGR.toString().getBytes());
scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type));
scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector));
scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs));
@@ -90,7 +90,6 @@ public class ObserverEnabler {
private static boolean isCoprocessorBeneficial(CubeInstance cube, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
-
String forceFlag = System.getProperty(FORCE_COPROCESSOR);
if (forceFlag != null) {
return Boolean.parseBoolean(forceFlag);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
index 6c789d3..9bed0ca 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
@@ -18,36 +18,36 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverAggregators.HCol;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverAggregators.HCol;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* @author yangli9
@@ -122,7 +122,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -171,7 +171,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner,ObserverBehavior.SCAN_FILTER_AGGR);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
[2/4] incubator-kylin git commit: KYLIN-844 add BackdoorToggles to
control fuzzy key
Posted by ma...@apache.org.
KYLIN-844 add BackdoorToggles to control fuzzy key
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e8132b73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e8132b73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e8132b73
Branch: refs/heads/0.7-staging
Commit: e8132b73ab00c1474f78821d62b240c51c25eadd
Parents: ad4a28a
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 12:02:06 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:16:52 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 53 ++++++++++++++++
.../org/apache/kylin/common/util/BytesUtil.java | 12 ++--
.../apache/kylin/common/util/BytesUtilTest.java | 9 +++
.../apache/kylin/job/tools/RowCounterCLI.java | 67 ++++++++++++++++++++
.../kylin/rest/controller/QueryController.java | 67 ++++++++++++--------
.../apache/kylin/rest/request/SQLRequest.java | 13 ++++
.../kylin/storage/hbase/HBaseKeyRange.java | 26 +++++---
.../coprocessor/observer/ObserverEnabler.java | 12 +++-
8 files changed, 218 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
new file mode 100644
index 0000000..315543f
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -0,0 +1,53 @@
+package org.apache.kylin.common.debug;
+
+import java.util.Map;
+
+/**
+ */
+public class BackdoorToggles {
+
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal();
+
+ public static void setToggles(Map<String, String> toggles) {
+ _backdoorToggles.set(toggles);
+ }
+
+ public static String getToggle(String key) {
+ Map<String, String> toggles = _backdoorToggles.get();
+ if (toggles == null) {
+ return null;
+ } else {
+ return toggles.get(key);
+ }
+ }
+
+ public static void cleanToggles() {
+ _backdoorToggles.remove();
+ }
+
+ /**
+ * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
+ *
+ *
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
+
+ /**
+ * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 41946ca..decdc3a 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -18,16 +18,15 @@
package org.apache.kylin.common.util;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.Writable;
+
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.io.Writable;
-
public class BytesUtil {
public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
@@ -371,6 +370,10 @@ public class BytesUtil {
return toHex(array);
}
+
+ /**
+ * this method only works for hex strings
+ */
public static byte[] fromReadableText(String text) {
String[] tokens = text.split("\\\\x");
byte[] ret = new byte[tokens.length - 1];
@@ -399,6 +402,7 @@ public class BytesUtil {
return sb.toString();
}
+
public static void main(String[] args) throws Exception {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
index 4c88cbe..2a3f0ef 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -53,4 +53,13 @@ public class BytesUtilTest extends TestCase {
assertEquals(y[1], false);
}
+ @Test
+ public void testReadable()
+ {
+ String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
+ byte[] bytes = BytesUtil.fromReadableText(x);
+ String y = BytesUtil.toHex(bytes);
+ assertEquals(x,y);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
new file mode 100644
index 0000000..b80063d
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
@@ -0,0 +1,67 @@
+package org.apache.kylin.job.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ */
+public class RowCounterCLI {
+ private static final Logger logger = LoggerFactory.getLogger(RowCounterCLI.class);
+
+ public static void main(String[] args) throws IOException {
+
+ if (args == null || args.length != 3) {
+ System.out.println("Usage: hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar org.apache.kylin.job.tools.RowCounterCLI [HTABLE_NAME] [STARTKEY] [ENDKEY]");
+ }
+
+ System.out.println(args[0]);
+ String htableName = args[0];
+ System.out.println(args[1]);
+ byte[] startKey = BytesUtil.fromReadableText(args[1]);
+ System.out.println(args[2]);
+ byte[] endKey = BytesUtil.fromReadableText(args[2]);
+
+ if (startKey == null) {
+ System.out.println("startkey is null ");
+ } else {
+ System.out.println("startkey lenght: " + startKey.length);
+ }
+
+ System.out.println("start key in binary: " + Bytes.toStringBinary(startKey));
+ System.out.println("end key in binary: " + Bytes.toStringBinary(endKey));
+
+ Configuration conf = HBaseConfiguration.create();
+
+ Scan scan = new Scan();
+ scan.setCaching(1024);
+ scan.setCacheBlocks(true);
+ scan.setStartRow(startKey);
+ scan.setStopRow(endKey);
+
+ logger.info("My Scan " + scan.toString());
+
+ HConnection conn = HConnectionManager.createConnection(conf);
+ HTableInterface tableInterface = conn.getTable(htableName);
+
+ Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
+ int counter = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ counter++;
+ if (counter % 1000 == 1) {
+ System.out.println("number of rows: " + counter);
+ }
+ }
+ System.out.println("number of rows: " + counter);
+ tableInterface.close();
+ conn.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 165660e..7cd14ff 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -18,50 +18,45 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import javax.servlet.http.HttpServletResponse;
-
+import com.codahale.metrics.annotation.Timed;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
-
import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.model.Query;
+import org.apache.kylin.rest.model.SelectedColumnMeta;
+import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.MetaRequest;
+import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.service.QueryService;
+import org.apache.kylin.rest.util.QueryUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
-import com.codahale.metrics.annotation.Timed;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.model.Query;
-import org.apache.kylin.rest.model.SelectedColumnMeta;
-import org.apache.kylin.rest.model.TableMeta;
-import org.apache.kylin.rest.request.PrepareSqlRequest;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.rest.request.SaveSqlRequest;
-import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryUtil;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
/**
* Handle query requests.
@@ -86,13 +81,15 @@ public class QueryController extends BasicController {
@ResponseBody
@Timed(name = "query")
public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
- long startTimestamp = System.currentTimeMillis();
+ initDebugToggles(sqlRequest);
+ long startTimestamp = System.currentTimeMillis();
SQLResponse response = doQuery(sqlRequest);
response.setDuration(System.currentTimeMillis() - startTimestamp);
-
queryService.logQuery(sqlRequest, response, new Date(startTimestamp), new Date(System.currentTimeMillis()));
+ cleanupDebugToggles();
+
return response;
}
@@ -261,4 +258,18 @@ public class QueryController extends BasicController {
this.cacheManager = cacheManager;
}
+ private void initDebugToggles(SQLRequest sqlRequest) {
+
+ Map<String, String> toggles = sqlRequest.getBackdoorToggles();
+ if (toggles == null || toggles.size() == 0) {
+ return;
+ }
+
+ BackdoorToggles.setToggles(toggles);
+ }
+
+ private void cleanupDebugToggles() {
+ BackdoorToggles.cleanToggles();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index a75ddfb..901badd 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.request;
+import java.util.Map;
+
public class SQLRequest {
private String sql;
@@ -26,9 +28,20 @@ public class SQLRequest {
private Integer limit = 0;
private boolean acceptPartial = true;
+ private Map<String,String> backdoorToggles;
+
public SQLRequest() {
}
+ public Map<String, String> getBackdoorToggles() {
+ return backdoorToggles;
+ }
+
+ public void setBackdoorToggles(Map<String, String> backdoorToggles) {
+ this.backdoorToggles = backdoorToggles;
+ }
+
+
public String getProject() {
return project;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index cb2dce7..278c50c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -18,12 +18,9 @@
package org.apache.kylin.storage.hbase;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.DateFormat;
@@ -36,9 +33,10 @@ import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.util.*;
/**
*
@@ -47,6 +45,8 @@ import com.google.common.collect.Maps;
*/
public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
+ private static final Logger logger = LoggerFactory.getLogger(HBaseKeyRange.class);
+
private static final int FUZZY_VALUE_CAP = 20;
private static final byte[] ZERO_TAIL_BYTES = new byte[] { 0 };
@@ -160,6 +160,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
+ buf.append(System.lineSeparator());
}
this.fuzzyKeyString = buf.toString();
}
@@ -167,6 +168,15 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) {
ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>();
+ //debug/profiling purpose
+ String toggle;
+ if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_DISABLE_FUZZY_KEY)) != null) {
+ if (Boolean.valueOf(toggle)) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
+ }
+ }
+
FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
FuzzyMaskEncoder fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index ce9ba9d..6acc40e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -72,8 +73,17 @@ public class ObserverEnabler {
AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
+
+ //debug/profiling purpose
+ String toggle;
+ if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_OBSERVER_BEHAVIOR)) == null) {
+ toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString();//default behavior
+ } else {
+ logger.info("The execution of this query will use " + toggle + " as observer's behavior");
+ }
+
scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 });
- scan.setAttribute(AggregateRegionObserver.BEHAVIOR, ObserverBehavior.SCAN_FILTER_AGGR.toString().getBytes());
+ scan.setAttribute(AggregateRegionObserver.BEHAVIOR, toggle.getBytes());
scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type));
scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector));
scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs));
[3/4] incubator-kylin git commit: hot fix: filter evaluation
performance issue caused by TableDesc
Posted by ma...@apache.org.
hot fix: filter evaluation performance issue caused by TableDesc
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e0733c96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e0733c96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e0733c96
Branch: refs/heads/0.7-staging
Commit: e0733c96db5ed908ecb2fead8067beb42d0fd445
Parents: e8132b7
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 21:20:25 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:20:25 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/TableDesc.java | 13 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 10 +-
.../kylin/storage/filter/FilterPerfTest.java | 132 +++++++++++++++++++
3 files changed, 146 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0733c96/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 14ea390..30c4d76 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -18,9 +18,6 @@
package org.apache.kylin.metadata.model;
-import java.util.Arrays;
-import java.util.Comparator;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -28,6 +25,9 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringSplitter;
+import java.util.Arrays;
+import java.util.Comparator;
+
/**
* Table Metadata from Source. All name should be uppercase.
* <p/>
@@ -43,6 +43,8 @@ public class TableDesc extends RootPersistentEntity {
private DatabaseDesc database;
+ private String identity = null;
+
public ColumnDesc findColumnByName(String name) {
//ignore the db name and table name if exists
int lastIndexOfDot = name.lastIndexOf(".");
@@ -73,7 +75,10 @@ public class TableDesc extends RootPersistentEntity {
}
public String getIdentity() {
- return String.format("%s.%s", this.getDatabase().toUpperCase(), this.getName()).toUpperCase();
+ if (identity == null) {
+ identity = String.format("%s.%s", this.getDatabase().toUpperCase(), this.getName()).toUpperCase();
+ }
+ return identity;
}
public static String concatResourcePath(String tableIdentity) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0733c96/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 5023580..92e5174 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -18,10 +18,6 @@
package org.apache.kylin.storage.tuple;
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.cube.CubeManager;
@@ -31,6 +27,10 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
/**
* @author xjiang
*/
@@ -82,7 +82,7 @@ public class Tuple implements ITuple {
return info.getDataType(fieldName);
}
- private void setFieldObjectValue(String fieldName, Object fieldValue) {
+ public void setFieldObjectValue(String fieldName, Object fieldValue) {
int index = info.getFieldIndex(fieldName);
values[index] = fieldValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0733c96/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java
new file mode 100644
index 0000000..e8826cd
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java
@@ -0,0 +1,132 @@
+package org.apache.kylin.storage.filter;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ */
+public class FilterPerfTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void foo() throws IOException, InterruptedException {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ TblColRef format = new TblColRef(tableDesc.findColumnByName("LSTG_FORMAT_NAME"));
+ TblColRef categ = new TblColRef(tableDesc.findColumnByName("LEAF_CATEG_ID"));
+ TblColRef site = new TblColRef(tableDesc.findColumnByName("LSTG_SITE_ID"));
+
+ List<TupleFilter> ands = Lists.newArrayList();
+ ands.add(buildOrFilterWithMultipleValues(format, Lists.newArrayList("Auction", "FP-GTC", "Others")));
+ ands.add(buildOrFilterWithMultipleValues(categ, Lists.newArrayList("48027", "164261", "113802", "118687")));
+ ands.add(buildOrFilterWithMultipleValues(site, Lists.newArrayList("0", "15", "3")));
+ TupleFilter filter = buildAndFilter(ands);
+
+ TupleInfo info = new TupleInfo();
+ ColumnDesc[] columns = tableDesc.getColumns();
+ for (int i = 0; i < columns.length; i++) {
+ ColumnDesc column = columns[i];
+ info.setField(column.getName(), new TblColRef(column), column.getDatatype(), i);
+ }
+
+ List<String> lines = Files.readLines(new File("../examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv"), Charset.defaultCharset());
+ List<Tuple> tuples = Lists.newArrayList();
+
+ for (String line : lines) {
+ String[] tokens = line.split(",");
+ if (tokens.length != columns.length) {
+ System.out.println("invalid line");
+ continue;
+ }
+ Tuple t = new Tuple(info);
+ for (int k = 0; k < columns.length; k++) {
+ ColumnDesc column = columns[k];
+ t.setFieldObjectValue(column.getName(), tokens[k]);
+ }
+ tuples.add(t);
+ }
+
+ for (int i = 0; i < 5; i++) {
+ Thread.sleep(1000);
+ System.out.println(i);
+ }
+
+ Iterator<Tuple> itr = Iterables.cycle(tuples).iterator();
+ int TOTAL_LOOP = 1000000;
+ int loopCount = 0;
+ int matchCount = 0;
+ long startTime = System.currentTimeMillis();
+ while (itr.hasNext()) {
+ if (filter.evaluate(itr.next())) {
+ matchCount++;
+ }
+
+ if (++loopCount > TOTAL_LOOP) {
+ break;
+ }
+ }
+ System.out.println("Total match count: " + matchCount);
+ System.out.println("ellapsed time: " + (System.currentTimeMillis() - startTime));
+ }
+
+ private TupleFilter buildOrFilterWithMultipleValues(TblColRef column, List<String> values) {
+ List<TupleFilter> ors = Lists.newArrayList();
+ for (String v : values) {
+ ors.add(buildCompareFilter(column, v));
+ }
+ return buildOrFilter(ors);
+ }
+
+ protected CompareTupleFilter buildCompareFilter(TblColRef column, String v) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = new ConstantTupleFilter(v);
+ compareFilter.addChild(constantFilter);
+ return compareFilter;
+ }
+
+ protected TupleFilter buildAndFilter(List<TupleFilter> ands) {
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ for (TupleFilter x : ands) {
+ andFilter.addChild(x);
+ }
+ return andFilter;
+ }
+
+ protected TupleFilter buildOrFilter(List<TupleFilter> ors) {
+ LogicalTupleFilter orFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ for (TupleFilter x : ors) {
+ orFilter.addChild(x);
+ }
+ return orFilter;
+ }
+
+}
[4/4] incubator-kylin git commit: hot fix: delete query cache at
wiping all cache
Posted by ma...@apache.org.
hot fix: delete query cache at wiping all cache
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/269d9225
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/269d9225
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/269d9225
Branch: refs/heads/0.7-staging
Commit: 269d92253907bb714f7d33a8daf3df4e08c4584f
Parents: e0733c9
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 21:20:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:20:53 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/CacheService.java | 119 ++++++++++---------
1 file changed, 60 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/269d9225/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index e104897..164ca12 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -41,44 +41,45 @@ public class CacheService extends BasicService {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().loadCubeCache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().reloadCubeDesc(cacheKey);
- break;
- case PROJECT:
- getProjectManager().reloadProject(cacheKey);
- break;
- case INVERTED_INDEX:
- getIIManager().loadIICache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().reloadIIDesc(cacheKey);
- break;
- case TABLE:
- getMetadataManager().reloadTableCache(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case DATA_MODEL:
- getMetadataManager().reloadDataModelDesc(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case ALL:
- getMetadataManager().reload();
- CubeDescManager.clearCache();
- CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
- ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
- break;
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().loadCubeCache(cacheKey);
+ cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().reloadCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ getProjectManager().reloadProject(cacheKey);
+ break;
+ case INVERTED_INDEX:
+ getIIManager().loadIICache(cacheKey);
+ cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().reloadIIDesc(cacheKey);
+ break;
+ case TABLE:
+ getMetadataManager().reloadTableCache(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case DATA_MODEL:
+ getMetadataManager().reloadDataModelDesc(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case ALL:
+ getMetadataManager().reload();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ IIDescManager.clearCache();
+ IIManager.clearCache();
+ ProjectManager.clearCache();
+ BasicService.resetOLAPDataSources();
+ cleanDataCache();
+ break;
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
@@ -98,27 +99,27 @@ public class CacheService extends BasicService {
final String log = "remove cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().removeCubeCacheLocal(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().removeLocalCubeDesc(cacheKey);
- break;
- case PROJECT:
- ProjectManager.clearCache();
- break;
- case INVERTED_INDEX:
- getIIManager().removeIILocalCache(cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().removeIIDescLocal(cacheKey);
- break;
- case TABLE:
- throw new UnsupportedOperationException(log);
- case DATA_MODEL:
- throw new UnsupportedOperationException(log);
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().removeCubeCacheLocal(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().removeLocalCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ ProjectManager.clearCache();
+ break;
+ case INVERTED_INDEX:
+ getIIManager().removeIILocalCache(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().removeIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ throw new UnsupportedOperationException(log);
+ case DATA_MODEL:
+ throw new UnsupportedOperationException(log);
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);