You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/24 15:23:29 UTC
[2/4] incubator-kylin git commit: KYLIN-844 add BackdoorToggles to
control fuzzy key
KYLIN-844 add BackdoorToggles to control fuzzy key
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e8132b73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e8132b73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e8132b73
Branch: refs/heads/0.7-staging
Commit: e8132b73ab00c1474f78821d62b240c51c25eadd
Parents: ad4a28a
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 12:02:06 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:16:52 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 53 ++++++++++++++++
.../org/apache/kylin/common/util/BytesUtil.java | 12 ++--
.../apache/kylin/common/util/BytesUtilTest.java | 9 +++
.../apache/kylin/job/tools/RowCounterCLI.java | 67 ++++++++++++++++++++
.../kylin/rest/controller/QueryController.java | 67 ++++++++++++--------
.../apache/kylin/rest/request/SQLRequest.java | 13 ++++
.../kylin/storage/hbase/HBaseKeyRange.java | 26 +++++---
.../coprocessor/observer/ObserverEnabler.java | 12 +++-
8 files changed, 218 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
new file mode 100644
index 0000000..315543f
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -0,0 +1,53 @@
+package org.apache.kylin.common.debug;
+
+import java.util.Map;
+
+/**
+ */
+public class BackdoorToggles {
+
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal();
+
+ public static void setToggles(Map<String, String> toggles) {
+ _backdoorToggles.set(toggles);
+ }
+
+ public static String getToggle(String key) {
+ Map<String, String> toggles = _backdoorToggles.get();
+ if (toggles == null) {
+ return null;
+ } else {
+ return toggles.get(key);
+ }
+ }
+
+ public static void cleanToggles() {
+ _backdoorToggles.remove();
+ }
+
+ /**
+ * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
+ *
+ *
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
+
+ /**
+ * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 41946ca..decdc3a 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -18,16 +18,15 @@
package org.apache.kylin.common.util;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.Writable;
+
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.io.Writable;
-
public class BytesUtil {
public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
@@ -371,6 +370,10 @@ public class BytesUtil {
return toHex(array);
}
+
+ /**
+ * this method only works for hex strings
+ */
public static byte[] fromReadableText(String text) {
String[] tokens = text.split("\\\\x");
byte[] ret = new byte[tokens.length - 1];
@@ -399,6 +402,7 @@ public class BytesUtil {
return sb.toString();
}
+
public static void main(String[] args) throws Exception {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
index 4c88cbe..2a3f0ef 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -53,4 +53,13 @@ public class BytesUtilTest extends TestCase {
assertEquals(y[1], false);
}
+ @Test
+ public void testReadable()
+ {
+ String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
+ byte[] bytes = BytesUtil.fromReadableText(x);
+ String y = BytesUtil.toHex(bytes);
+ assertEquals(x,y);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
new file mode 100644
index 0000000..b80063d
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
@@ -0,0 +1,67 @@
+package org.apache.kylin.job.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ */
+public class RowCounterCLI {
+ private static final Logger logger = LoggerFactory.getLogger(RowCounterCLI.class);
+
+ public static void main(String[] args) throws IOException {
+
+ if (args == null || args.length != 3) {
+ System.out.println("Usage: hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar org.apache.kylin.job.tools.RowCounterCLI [HTABLE_NAME] [STARTKEY] [ENDKEY]");
+ }
+
+ System.out.println(args[0]);
+ String htableName = args[0];
+ System.out.println(args[1]);
+ byte[] startKey = BytesUtil.fromReadableText(args[1]);
+ System.out.println(args[2]);
+ byte[] endKey = BytesUtil.fromReadableText(args[2]);
+
+ if (startKey == null) {
+ System.out.println("startkey is null ");
+ } else {
+ System.out.println("startkey lenght: " + startKey.length);
+ }
+
+ System.out.println("start key in binary: " + Bytes.toStringBinary(startKey));
+ System.out.println("end key in binary: " + Bytes.toStringBinary(endKey));
+
+ Configuration conf = HBaseConfiguration.create();
+
+ Scan scan = new Scan();
+ scan.setCaching(1024);
+ scan.setCacheBlocks(true);
+ scan.setStartRow(startKey);
+ scan.setStopRow(endKey);
+
+ logger.info("My Scan " + scan.toString());
+
+ HConnection conn = HConnectionManager.createConnection(conf);
+ HTableInterface tableInterface = conn.getTable(htableName);
+
+ Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
+ int counter = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ counter++;
+ if (counter % 1000 == 1) {
+ System.out.println("number of rows: " + counter);
+ }
+ }
+ System.out.println("number of rows: " + counter);
+ tableInterface.close();
+ conn.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 165660e..7cd14ff 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -18,50 +18,45 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import javax.servlet.http.HttpServletResponse;
-
+import com.codahale.metrics.annotation.Timed;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
-
import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.model.Query;
+import org.apache.kylin.rest.model.SelectedColumnMeta;
+import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.MetaRequest;
+import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.service.QueryService;
+import org.apache.kylin.rest.util.QueryUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
-import com.codahale.metrics.annotation.Timed;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.model.Query;
-import org.apache.kylin.rest.model.SelectedColumnMeta;
-import org.apache.kylin.rest.model.TableMeta;
-import org.apache.kylin.rest.request.PrepareSqlRequest;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.rest.request.SaveSqlRequest;
-import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryUtil;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
/**
* Handle query requests.
@@ -86,13 +81,15 @@ public class QueryController extends BasicController {
@ResponseBody
@Timed(name = "query")
public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
- long startTimestamp = System.currentTimeMillis();
+ initDebugToggles(sqlRequest);
+ long startTimestamp = System.currentTimeMillis();
SQLResponse response = doQuery(sqlRequest);
response.setDuration(System.currentTimeMillis() - startTimestamp);
-
queryService.logQuery(sqlRequest, response, new Date(startTimestamp), new Date(System.currentTimeMillis()));
+ cleanupDebugToggles();
+
return response;
}
@@ -261,4 +258,18 @@ public class QueryController extends BasicController {
this.cacheManager = cacheManager;
}
+ private void initDebugToggles(SQLRequest sqlRequest) {
+
+ Map<String, String> toggles = sqlRequest.getBackdoorToggles();
+ if (toggles == null || toggles.size() == 0) {
+ return;
+ }
+
+ BackdoorToggles.setToggles(toggles);
+ }
+
+ private void cleanupDebugToggles() {
+ BackdoorToggles.cleanToggles();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index a75ddfb..901badd 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.request;
+import java.util.Map;
+
public class SQLRequest {
private String sql;
@@ -26,9 +28,20 @@ public class SQLRequest {
private Integer limit = 0;
private boolean acceptPartial = true;
+ private Map<String,String> backdoorToggles;
+
public SQLRequest() {
}
+ public Map<String, String> getBackdoorToggles() {
+ return backdoorToggles;
+ }
+
+ public void setBackdoorToggles(Map<String, String> backdoorToggles) {
+ this.backdoorToggles = backdoorToggles;
+ }
+
+
public String getProject() {
return project;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index cb2dce7..278c50c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -18,12 +18,9 @@
package org.apache.kylin.storage.hbase;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.DateFormat;
@@ -36,9 +33,10 @@ import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.util.*;
/**
*
@@ -47,6 +45,8 @@ import com.google.common.collect.Maps;
*/
public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
+ private static final Logger logger = LoggerFactory.getLogger(HBaseKeyRange.class);
+
private static final int FUZZY_VALUE_CAP = 20;
private static final byte[] ZERO_TAIL_BYTES = new byte[] { 0 };
@@ -160,6 +160,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
+ buf.append(System.lineSeparator());
}
this.fuzzyKeyString = buf.toString();
}
@@ -167,6 +168,15 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) {
ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>();
+ //debug/profiling purpose
+ String toggle;
+ if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_DISABLE_FUZZY_KEY)) != null) {
+ if (Boolean.valueOf(toggle)) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
+ }
+ }
+
FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
FuzzyMaskEncoder fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index ce9ba9d..6acc40e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -72,8 +73,17 @@ public class ObserverEnabler {
AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
+
+ //debug/profiling purpose
+ String toggle;
+ if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_OBSERVER_BEHAVIOR)) == null) {
+ toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString();//default behavior
+ } else {
+ logger.info("The execution of this query will use " + toggle + " as observer's behavior");
+ }
+
scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 });
- scan.setAttribute(AggregateRegionObserver.BEHAVIOR, ObserverBehavior.SCAN_FILTER_AGGR.toString().getBytes());
+ scan.setAttribute(AggregateRegionObserver.BEHAVIOR, toggle.getBytes());
scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type));
scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector));
scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs));