You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/05/11 12:26:31 UTC
kylin git commit: KYLIN-2610 Optimize BuiltInFunctionTransformer
performance
Repository: kylin
Updated Branches:
refs/heads/2.0.x c0392b778 -> 949c956ca
KYLIN-2610 Optimize BuiltInFunctionTransformer performance
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/949c956c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/949c956c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/949c956c
Branch: refs/heads/2.0.x
Commit: 949c956ca331e6ae9234fe6b7ba05e3234086eaa
Parents: c0392b7
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu May 11 20:26:14 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu May 11 20:26:14 2017 +0800
----------------------------------------------------------------------
.../kylin/dict/BuiltInFunctionTransformer.java | 6 +-
.../metadata/filter/function/BuiltInMethod.java | 39 ++++--
.../metadata/filter/function/LikeMatchers.java | 122 +++++++++++++++++++
3 files changed, 156 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/949c956c/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 4d7b15d..3d4bfe4 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -58,13 +58,13 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
//normal case
translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
if (translated != null) {
- logger.debug("Translated {{}} to IN clause: {{}}", tupleFilter, translated);
+ logger.debug("Translated {{}} to IN clause. ", tupleFilter);
}
} else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
- //like case
+ //like,tolower case
translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
if (translated != null) {
- logger.debug("Translated {{}} to IN clause: {{}}", tupleFilter, translated);
+ logger.debug("Translated {{}} to IN clause. ", tupleFilter);
}
} else if (tupleFilter instanceof LogicalTupleFilter) {
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/kylin/blob/949c956c/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
index 2f28fae..31ee297 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
@@ -18,8 +18,11 @@
package org.apache.kylin.metadata.filter.function;
+import static org.apache.kylin.metadata.filter.function.LikeMatchers.LikeMatcher;
+
import java.lang.reflect.Method;
-import java.util.regex.Pattern;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.lang3.reflect.MethodUtils;
@@ -28,9 +31,15 @@ import com.google.common.collect.ImmutableMap;
public enum BuiltInMethod {
UPPER(BuiltInMethod.class, "upper", String.class), LOWER(BuiltInMethod.class, "lower", String.class), SUBSTRING(BuiltInMethod.class, "substring", String.class, int.class, int.class), CHAR_LENGTH(BuiltInMethod.class, "charLength", String.class), LIKE(BuiltInMethod.class, "like", String.class, String.class), INITCAP(BuiltInMethod.class, "initcap", String.class);
public final Method method;
-
public static final ImmutableMap<String, BuiltInMethod> MAP;
+ private static ThreadLocal<Map<String, LikeMatcher>> likePatterns = new ThreadLocal<Map<String, LikeMatcher>>() {
+ @Override
+ public Map<String, LikeMatcher> initialValue() {
+ return new HashMap<>();
+ }
+ };
+
static {
final ImmutableMap.Builder<String, BuiltInMethod> builder = ImmutableMap.builder();
for (BuiltInMethod value : BuiltInMethod.values()) {
@@ -46,12 +55,26 @@ public enum BuiltInMethod {
}
/** SQL {@code LIKE} function. */
- public static boolean like(String s, String pattern) {
- if (s == null)
+ public static boolean like(String s, String patternStr) {
+ //TODO: escape in like is unsupported
+ //TODO: like is case sensitive now
+
+ if (s == null || patternStr == null)
return false;
-
- final String regex = Like.sqlToRegexLike(pattern, null);
- return Pattern.matches(regex, s);
+
+ Map<String, LikeMatcher> patterns = likePatterns.get();
+ LikeMatcher p = patterns.get(patternStr);
+ if (p == null) {
+
+ p = LikeMatchers.createMatcher(patternStr);
+
+ if (patterns.size() > 100) {
+ patterns.clear();//brutal but good enough
+ }
+ patterns.put(patternStr, p);
+ }
+
+ return p.matches(s);
}
/** SQL INITCAP(string) function. */
@@ -117,4 +140,4 @@ public enum BuiltInMethod {
return s.toLowerCase();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/949c956c/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java
new file mode 100644
index 0000000..c0b9bc2
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java
@@ -0,0 +1,122 @@
+package org.apache.kylin.metadata.filter.function;
+
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.base.Preconditions;
+
+public class LikeMatchers {
+
+ private final static String PERCENT_SIGN = "%";
+ private final static String UNDERSCORE_SIGN = "_";
+
+ public interface LikeMatcher {
+ boolean matches(String input);
+ }
+
+ public static class DefaultLikeMatcher implements LikeMatcher {
+ private Pattern p;
+
+ private DefaultLikeMatcher(String patternStr) {
+ patternStr = patternStr.toLowerCase();
+ final String regex = Like.sqlToRegexLike(patternStr, null);
+ p = Pattern.compile(regex);
+ }
+
+ @Override
+ public boolean matches(String input) {
+ return p.matcher(input).matches();
+ }
+ }
+
+ // abc%, %abc, abc%def
+ public static class OnePercentSignLikeMatcher implements LikeMatcher {
+
+ enum SignPosition {
+ LEFT, MIDDLE, RIGHT
+ }
+
+ private SignPosition signPosition;
+ private String[] remaining;
+
+ private OnePercentSignLikeMatcher(String patternStr) {
+ if (patternStr.startsWith(PERCENT_SIGN)) {
+ signPosition = SignPosition.LEFT;
+ } else if (patternStr.endsWith(PERCENT_SIGN)) {
+ signPosition = SignPosition.RIGHT;
+ } else {
+ signPosition = SignPosition.MIDDLE;
+ }
+ remaining = StringUtils.split(patternStr, PERCENT_SIGN);
+ }
+
+ @Override
+ public boolean matches(String input) {
+ if (input == null)
+ return false;
+
+ switch (signPosition) {
+ case LEFT:
+ return input.endsWith(remaining[0]);
+ case RIGHT:
+ return input.startsWith(remaining[0]);
+ case MIDDLE:
+ return input.startsWith(remaining[0]) && input.endsWith(remaining[1]);
+ default:
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ //only deal with %abc%
+ public static class TwoPercentSignLikeMatcher implements LikeMatcher {
+ private String[] remaining;
+
+ private TwoPercentSignLikeMatcher(String patternStr) {
+ remaining = StringUtils.split(patternStr, PERCENT_SIGN);
+ Preconditions.checkState(remaining.length == 1);
+ }
+
+ @Override
+ public boolean matches(String input) {
+ return input.contains(remaining[0]);
+ }
+ }
+
+ //only deal with %abc%def%
+ public static class ThreePercentSignLikeMatcher implements LikeMatcher {
+ private String[] remaining;
+
+ private ThreePercentSignLikeMatcher(String patternStr) {
+ remaining = StringUtils.split(patternStr, PERCENT_SIGN);
+ Preconditions.checkState(remaining.length == 2);
+ }
+
+ @Override
+ public boolean matches(String input) {
+ return input.indexOf(remaining[0]) < input.indexOf(remaining[1]);
+ }
+ }
+
+ public static LikeMatcher createMatcher(String patternStr) {
+ if (patternStr == null) {
+ throw new IllegalArgumentException("pattern is null");
+ }
+
+ if (patternStr.contains(UNDERSCORE_SIGN)) {
+ return new DefaultLikeMatcher(patternStr);
+ }
+
+ int count = StringUtils.countMatches(patternStr, PERCENT_SIGN);
+ if (count == 1) {
+ return new OnePercentSignLikeMatcher(patternStr);
+ } else if (count == 2 && patternStr.startsWith(PERCENT_SIGN) && patternStr.endsWith(PERCENT_SIGN)) {
+ return new TwoPercentSignLikeMatcher(patternStr);
+ } else if (count == 3 && patternStr.startsWith(PERCENT_SIGN) && patternStr.endsWith(PERCENT_SIGN) && !patternStr.contains(PERCENT_SIGN + PERCENT_SIGN)) {
+ return new ThreePercentSignLikeMatcher(patternStr);
+ } else {
+ return new DefaultLikeMatcher(patternStr);
+ }
+ }
+}