You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2013/09/23 04:26:04 UTC
git commit: TAJO-193: Add string pattern matching operators. (hyunsik)
Updated Branches:
refs/heads/master 16d1895e5 -> 7b0dec6a7
TAJO-193: Add string pattern matching operators. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/7b0dec6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/7b0dec6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/7b0dec6a
Branch: refs/heads/master
Commit: 7b0dec6a7b44fbacf582f8e8fd78d68064b42a7c
Parents: 16d1895
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Sep 23 11:25:29 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Sep 23 11:25:29 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tajo/algebra/LikePredicate.java | 51 -------
.../java/org/apache/tajo/algebra/OpType.java | 6 +-
.../tajo/algebra/PatternMatchPredicate.java | 69 +++++++++
.../org/apache/tajo/datum/BooleanDatum.java | 2 +-
.../org/apache/tajo/datum/DatumFactory.java | 2 +-
.../java/org/apache/tajo/datum/NullDatum.java | 12 +-
.../org/apache/tajo/datum/TestBoolDatum.java | 2 +-
.../org/apache/tajo/engine/parser/SQLLexer.g4 | 13 ++
.../org/apache/tajo/engine/parser/SQLParser.g4 | 30 +++-
.../tajo/engine/eval/BasicEvalNodeVisitor.java | 24 ++-
.../tajo/engine/eval/EvalNodeVisitor2.java | 6 +-
.../org/apache/tajo/engine/eval/EvalType.java | 8 +-
.../org/apache/tajo/engine/eval/LikeEval.java | 96 ------------
.../tajo/engine/eval/LikePredicateEval.java | 49 +++++++
.../engine/eval/PatternMatchPredicateEval.java | 90 ++++++++++++
.../tajo/engine/eval/RegexPredicateEval.java | 53 +++++++
.../engine/eval/SimilarToPredicateEval.java | 43 ++++++
.../tajo/engine/parser/HiveConverter.java | 15 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 48 +++++-
.../tajo/engine/planner/LogicalPlanner.java | 29 +++-
.../apache/tajo/engine/eval/TestEvalTree.java | 5 +-
.../tajo/engine/function/ExprTestBase.java | 145 +++++++++++++++++++
.../function/TestPatternMatchingPredicates.java | 137 ++++++++++++++++++
.../tajo/engine/query/TestInsertQuery.java | 16 +-
.../tajo/engine/query/TestSelectQuery.java | 14 +-
.../java/org/apache/tajo/storage/LazyTuple.java | 2 -
27 files changed, 767 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8b91f00..123ee14 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@ Release 0.2.0 - unreleased
NEW FEATURES
+ TAJO-193: Add string pattern matching operators. (hyunsik)
+
TAJO-101: HiveQL converter. (jaehwa)
TAJO-144: Implement INSERT OVERWRITE clause. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-algebra/src/main/java/org/apache/tajo/algebra/LikePredicate.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/LikePredicate.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/LikePredicate.java
deleted file mode 100644
index e091df1..0000000
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/LikePredicate.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.algebra;
-
-public class LikePredicate extends Expr {
- private boolean not;
- private ColumnReferenceExpr columnRef;
- private Expr pattern;
-
- public LikePredicate(boolean not, ColumnReferenceExpr columnReferenceExpr, Expr pattern) {
- super(OpType.LikePredicate);
- this.not = not;
- this.columnRef = columnReferenceExpr;
- this.pattern = pattern;
- }
-
- public boolean isNot() {
- return not;
- }
-
- public ColumnReferenceExpr getColumnRef() {
- return this.columnRef;
- }
-
- public Expr getPattern() {
- return this.pattern;
- }
-
- boolean equalsTo(Expr expr) {
- LikePredicate another = (LikePredicate) expr;
- return not == another.not &&
- columnRef.equals(another.columnRef) &&
- pattern.equals(another.pattern);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
index 7f46438..9b03fc0 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
@@ -64,10 +64,14 @@ public enum OpType {
CaseWhen(CaseWhenPredicate.class),
IsNullPredicate(IsNullPredicate.class),
InPredicate(InPredicate.class),
- LikePredicate(LikePredicate.class),
ValueList(ValueListExpr.class),
Is,
+ // pattern matching predicates
+ LikePredicate(PatternMatchPredicate.class),
+ SimilarToPredicate(PatternMatchPredicate.class),
+ Regexp(PatternMatchPredicate.class),
+
// arithmetic operators
Plus(BinaryOperator.class),
Minus(BinaryOperator.class),
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-algebra/src/main/java/org/apache/tajo/algebra/PatternMatchPredicate.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/PatternMatchPredicate.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/PatternMatchPredicate.java
new file mode 100644
index 0000000..e3c7ba4
--- /dev/null
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/PatternMatchPredicate.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.algebra;
+
+import com.google.common.base.Preconditions;
+
+public class PatternMatchPredicate extends Expr {
+ private boolean not;
+ private Expr columnRef;
+ private Expr pattern;
+ private boolean caseInsensitive;
+
+ public PatternMatchPredicate(OpType opType, boolean not, Expr predicand, Expr pattern,
+ boolean caseInsensitive) {
+ super(opType);
+ Preconditions.checkArgument(
+ opType == OpType.LikePredicate || opType == OpType.SimilarToPredicate || opType == OpType.Regexp,
+ "pattern matching predicate is only available: " + opType.name());
+ this.not = not;
+ this.columnRef = predicand;
+ this.pattern = pattern;
+ this.caseInsensitive = caseInsensitive;
+ }
+
+ public PatternMatchPredicate(OpType opType, boolean not, Expr predicand, Expr pattern) {
+ this(opType, not, predicand, pattern, false);
+ }
+
+ public boolean isNot() {
+ return not;
+ }
+
+ public Expr getPredicand() {
+ return this.columnRef;
+ }
+
+ public Expr getPattern() {
+ return this.pattern;
+ }
+
+ public boolean isCaseInsensitive() {
+ return this.caseInsensitive;
+ }
+
+ boolean equalsTo(Expr expr) {
+ PatternMatchPredicate another = (PatternMatchPredicate) expr;
+ return opType == another.opType &&
+ not == another.not &&
+ columnRef.equals(another.columnRef) &&
+ pattern.equals(another.pattern) &&
+ caseInsensitive == another.caseInsensitive;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
index fc543b4..e7c4d39 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
@@ -117,7 +117,7 @@ public class BooleanDatum extends Datum {
*/
@Override
public String asChars() {
- return val ? "true" : "false";
+ return val ? "t" : "f";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index 4563779..cc0089c 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -98,7 +98,7 @@ public class DatumFactory {
}
public static BooleanDatum createBool(String val) {
- boolean boolVal = val.equalsIgnoreCase("true");
+ boolean boolVal = val.equalsIgnoreCase("t");
return new BooleanDatum(boolVal);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
index 500f402..71d08d3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
@@ -24,7 +24,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type;
public class NullDatum extends Datum {
private static final NullDatum instance;
- private static String NULL_STRING = "null";
+ private static String NULL_STRING = "";
private static byte[] NULL_CHAR = NULL_STRING.getBytes();
static {
@@ -51,17 +51,17 @@ public class NullDatum extends Datum {
@Override
public short asInt2() {
- return Short.MIN_VALUE;
+ return 0;
}
@Override
public int asInt4() {
- return Integer.MIN_VALUE;
+ return 0;
}
@Override
public long asInt8() {
- return Long.MIN_VALUE;
+ return 0;
}
@Override
@@ -71,12 +71,12 @@ public class NullDatum extends Datum {
@Override
public float asFloat4() {
- return Float.NaN;
+ return 0f;
}
@Override
public double asFloat8() {
- return Double.NaN;
+ return 0d;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-common/src/test/java/org/apache/tajo/datum/TestBoolDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestBoolDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestBoolDatum.java
index 3623f88..3175510 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestBoolDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestBoolDatum.java
@@ -65,7 +65,7 @@ public class TestBoolDatum {
@Test
public final void testAsChars() {
Datum d = DatumFactory.createBool(true);
- assertEquals("true", d.asChars());
+ assertEquals("t", d.asChars());
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
index 627829e..4a32cf2 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
@@ -155,6 +155,7 @@ GROUPING : G R O U P I N G;
HAVING : H A V I N G;
+ILIKE : I L I K E;
IN : I N;
INDEX : I N D E X;
INNER : I N N E R;
@@ -188,7 +189,9 @@ OVERWRITE : O V E R W R I T E;
PRECISION : P R E C I S I ON;
+REGEXP : R E G E X P;
RIGHT : R I G H T;
+RLIKE : R L I K E;
ROLLUP : R O L L U P;
SET : S E T;
@@ -227,17 +230,22 @@ FUSION : F U S I O N;
INTERSECTION : I N T E R S E C T I O N;
+SIMILAR : S I M I L A R;
STDDEV_POP : S T D D E V UNDERLINE P O P;
STDDEV_SAMP : S T D D E V UNDERLINE S A M P;
SUM : S U M;
+TO : T O;
+
Nonreserved_keywords
: COLLECT
| FUSION
| INTERSECTION
+ | SIMILAR
| STDDEV_POP
| STDDEV_SAMP
| SUM
+ | TO
;
/*
@@ -292,6 +300,11 @@ BYTEA : B Y T E A; // alias for BLOB
INET4 : I N E T '4';
// Operators
+Similar_To : '~';
+Not_Similar_To : '!~';
+Similar_To_Case_Insensitive : '~*';
+Not_Similar_To_Case_Insensitive : '!~*';
+
ASSIGN : ':=';
EQUAL : '=';
SEMI_COLON : ';';
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index e4256bf..049d447 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -633,7 +633,7 @@ boolean_primary
predicate
: comparison_predicate
| in_predicate
- | like_predicate
+ | pattern_matching_predicate
| null_predicate
;
@@ -681,14 +681,34 @@ in_value_list
/*
===============================================================================
- <like_predicate>
+ <pattern_matching_predicate>
- Specify a pattern-match comparison.
+ Specify a pattern-matching comparison.
===============================================================================
*/
-like_predicate
- : f=column_reference NOT? LIKE s=Character_String_Literal
+pattern_matching_predicate
+ : f=numeric_primary pattern_matcher s=Character_String_Literal
+ ;
+
+pattern_matcher
+ : NOT? negativable_matcher
+ | regex_matcher
+ ;
+
+negativable_matcher
+ : LIKE
+ | ILIKE
+ | SIMILAR TO
+ | REGEXP
+ | RLIKE
+ ;
+
+regex_matcher
+ : Similar_To
+ | Not_Similar_To
+ | Similar_To_Case_Insensitive
+ | Not_Similar_To_Case_Insensitive
;
/*
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
index 8d6e250..7e34c5a 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
@@ -84,7 +84,7 @@ public class BasicEvalNodeVisitor<CONTEXT, RESULT> implements EvalNodeVisitor2<C
result = visitGreaterThanOrEqual(context, stack, (BinaryEval) evalNode);
break;
- // Other Predicates
+ // SQL standard predicates
case IS_NULL:
result = visitIsNull(context, stack, (IsNullEval) evalNode);
break;
@@ -97,8 +97,16 @@ public class BasicEvalNodeVisitor<CONTEXT, RESULT> implements EvalNodeVisitor2<C
case IN:
result = visitInPredicate(context, stack, (InEval) evalNode);
break;
+
+ // Pattern match predicates
case LIKE:
- result = visitLike(context, stack, (LikeEval) evalNode);
+ result = visitLike(context, stack, (LikePredicateEval) evalNode);
+ break;
+ case SIMILAR_TO:
+ result = visitSimilarTo(context, stack, (SimilarToPredicateEval) evalNode);
+ break;
+ case Regex:
+ result = visitRegex(context, stack, (RegexPredicateEval) evalNode);
break;
// Functions
@@ -264,7 +272,17 @@ public class BasicEvalNodeVisitor<CONTEXT, RESULT> implements EvalNodeVisitor2<C
}
@Override
- public RESULT visitLike(CONTEXT context, Stack<EvalNode> stack, LikeEval evalNode) {
+ public RESULT visitLike(CONTEXT context, Stack<EvalNode> stack, LikePredicateEval evalNode) {
+ return visitDefaultBinaryEval(context, stack, evalNode);
+ }
+
+ @Override
+ public RESULT visitSimilarTo(CONTEXT context, Stack<EvalNode> stack, SimilarToPredicateEval evalNode) {
+ return visitDefaultBinaryEval(context, stack, evalNode);
+ }
+
+ @Override
+ public RESULT visitRegex(CONTEXT context, Stack<EvalNode> stack, RegexPredicateEval evalNode) {
return visitDefaultBinaryEval(context, stack, evalNode);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java
index abb6b51..5c48008 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java
@@ -53,7 +53,11 @@ public interface EvalNodeVisitor2<CONTEXT, RESULT> {
RESULT visitCaseWhen(CONTEXT context, Stack<EvalNode> stack, CaseWhenEval evalNode);
RESULT visitIfThen(CONTEXT context, Stack<EvalNode> stack, CaseWhenEval.IfThenEval evalNode);
RESULT visitInPredicate(CONTEXT context, Stack<EvalNode> stack, InEval evalNode);
- RESULT visitLike(CONTEXT context, Stack<EvalNode> stack, LikeEval evalNode);
+
+ // Pattern matching predicates
+ RESULT visitLike(CONTEXT context, Stack<EvalNode> stack, LikePredicateEval evalNode);
+ RESULT visitSimilarTo(CONTEXT context, Stack<EvalNode> stack, SimilarToPredicateEval evalNode);
+ RESULT visitRegex(CONTEXT context, Stack<EvalNode> stack, RegexPredicateEval evalNode);
// Functions
RESULT visitFuncCall(CONTEXT context, Stack<EvalNode> stack, FuncCallEval evalNode);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
index 2e6742f..e10017e 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
@@ -42,8 +42,12 @@ public enum EvalType {
AGG_FUNCTION(AggFuncCallEval.class),
FUNCTION(FuncCallEval.class),
- // Predicate
- LIKE(LikeEval.class),
+ // String pattern matching
+ LIKE(LikePredicateEval.class),
+ SIMILAR_TO(SimilarToPredicateEval.class),
+ Regex(RegexPredicateEval.class),
+
+ // Other predicates
CASE(CaseWhenEval.class),
IF_THEN(CaseWhenEval.IfThenEval.class),
IN(InEval.class),
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikeEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikeEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikeEval.java
deleted file mode 100644
index d3ab277..0000000
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikeEval.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.engine.eval;
-
-import com.google.gson.annotations.Expose;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.common.TajoDataTypes.DataType;
-import org.apache.tajo.datum.BooleanDatum;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.datum.TextDatum;
-import org.apache.tajo.storage.Tuple;
-
-import java.util.regex.Pattern;
-
-public class LikeEval extends BinaryEval {
- @Expose private boolean not;
- @Expose private Column column;
- @Expose private String pattern;
- private static final DataType [] RES_TYPE =
- CatalogUtil.newDataTypesWithoutLen(TajoDataTypes.Type.BOOLEAN);
-
- // temporal variables
- private Integer fieldId = null;
- private Pattern compiled;
- private BooleanDatum result;
-
-
- public LikeEval(boolean not, FieldEval field, ConstEval pattern) {
- super(EvalType.LIKE, field, pattern);
- this.not = not;
- this.column = field.getColumnRef();
- this.pattern = pattern.getValue().asChars();
- }
-
- public void compile(String pattern) {
- String regex = pattern.replace("?", ".");
- regex = regex.replace("%", ".*");
-
- this.compiled = Pattern.compile(regex,
- Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
- result = DatumFactory.createBool(false);
- }
-
- @Override
- public DataType [] getValueType() {
- return RES_TYPE;
- }
-
- @Override
- public String getName() {
- return "?";
- }
-
- @Override
- public void eval(EvalContext ctx, Schema schema, Tuple tuple) {
- if (fieldId == null) {
- fieldId = schema.getColumnId(column.getQualifiedName());
- compile(this.pattern);
- }
- TextDatum str = tuple.getString(fieldId);
- if (not) {
- result.setValue(!compiled.matcher(str.asChars()).matches());
- } else {
- result.setValue(compiled.matcher(str.asChars()).matches());
- }
- }
-
- public Datum terminate(EvalContext ctx) {
- return result;
- }
-
- @Override
- public String toString() {
- return this.column + " like '" + pattern +"'";
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java
new file mode 100644
index 0000000..ac7aeeb
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.eval;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class LikePredicateEval extends PatternMatchPredicateEval {
+ private static final String LIKE_ESCAPE_SPATIAL_CHARACTERS = "([.*${}?|\\^\\-\\[\\]])";
+
+ public LikePredicateEval(boolean not, EvalNode field, ConstEval pattern, boolean caseSensitive) {
+ super(EvalType.LIKE, not, field, pattern, caseSensitive);
+ }
+
+ private String escapeRegexpForLike(String literal) {
+ return literal.replaceAll(LIKE_ESCAPE_SPATIAL_CHARACTERS, "\\\\$1");
+ }
+
+ protected void compile(String pattern) throws PatternSyntaxException {
+ String escaped = escapeRegexpForLike(pattern);
+ String regex = escaped.replace("_", ".").replace("%", ".*");
+ int flags = Pattern.DOTALL;
+ if (caseInsensitive) {
+ flags |= Pattern.CASE_INSENSITIVE;
+ }
+ this.compiled = Pattern.compile(regex, flags);
+ }
+
+ @Override
+ public String toString() {
+ return leftExpr.toString() + (caseInsensitive ? "ILIKE" : "LIKE") + "'" + pattern +"'";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java
new file mode 100644
index 0000000..83ee1e7
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.eval;
+
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.common.TajoDataTypes.DataType;
+import org.apache.tajo.datum.BooleanDatum;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.storage.Tuple;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public abstract class PatternMatchPredicateEval extends BinaryEval {
+ private static final DataType [] RES_TYPE = CatalogUtil.newDataTypesWithoutLen(TajoDataTypes.Type.BOOLEAN);
+
+ @Expose protected boolean not;
+ @Expose protected String pattern;
+ @Expose protected boolean caseInsensitive;
+
+ // transient variables
+ private EvalContext leftContext;
+ private boolean isNullResult = false;
+ private BooleanDatum result;
+ protected Pattern compiled;
+
+ public PatternMatchPredicateEval(EvalType evalType, boolean not, EvalNode predicand, ConstEval pattern,
+ boolean caseInsensitive) {
+ super(evalType, predicand, pattern);
+ this.not = not;
+ this.pattern = pattern.getValue().asChars();
+ this.caseInsensitive = caseInsensitive;
+ }
+
+ public PatternMatchPredicateEval(EvalType evalType, boolean not, EvalNode field, ConstEval pattern) {
+ this(evalType, not, field, pattern, false);
+ }
+
+ abstract void compile(String pattern) throws PatternSyntaxException;
+
+ @Override
+ public DataType [] getValueType() {
+ return RES_TYPE;
+ }
+
+ @Override
+ public String getName() {
+ return "?";
+ }
+
+ @Override
+ public void eval(EvalContext ctx, Schema schema, Tuple tuple) {
+ if (leftContext == null) {
+ leftContext = leftExpr.newContext();
+ result = DatumFactory.createBool(false);
+ compile(this.pattern);
+ }
+
+ leftExpr.eval(leftContext, schema, tuple);
+ Datum predicand = leftExpr.terminate(leftContext);
+ isNullResult = predicand instanceof NullDatum;
+ boolean matched = compiled.matcher(predicand.asChars()).matches();
+ result.setValue(matched ^ not);
+ }
+
+ public Datum terminate(EvalContext ctx) {
+ return !isNullResult ? result : NullDatum.get();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
new file mode 100644
index 0000000..a173d8f
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.eval;
+
+import com.google.gson.annotations.Expose;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class RegexPredicateEval extends PatternMatchPredicateEval {
+ @Expose private String operator;
+ public RegexPredicateEval(boolean not, EvalNode field, ConstEval pattern, boolean caseInsensitive) {
+ super(EvalType.Regex, not, field, pattern, caseInsensitive);
+ StringBuilder sb = new StringBuilder();
+ if (not) {
+ sb.append("!");
+ }
+ sb.append("~");
+ if (caseInsensitive) {
+ sb.append("*");
+ }
+ this.operator = sb.toString();
+ }
+
+ protected void compile(String regex) throws PatternSyntaxException {
+ int flags = Pattern.DOTALL;
+ if (caseInsensitive) {
+ flags |= Pattern.CASE_INSENSITIVE;
+ }
+ this.compiled = Pattern.compile(regex, flags);
+ }
+
+ @Override
+ public String toString() {
+ return leftExpr.toString() + operator + "'" + pattern +"'";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java
new file mode 100644
index 0000000..9ac0e62
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.eval;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class SimilarToPredicateEval extends PatternMatchPredicateEval {
+ private static final String SIMILARTO_ESCAPE_SPATIAL_CHARACTERS = "([.])";
+
+ public SimilarToPredicateEval(boolean not, EvalNode field, ConstEval pattern) {
+ super(EvalType.SIMILAR_TO, not, field, pattern);
+ }
+
+ @Override
+ protected void compile(String pattern) throws PatternSyntaxException {
+ String regex = pattern.replaceAll(SIMILARTO_ESCAPE_SPATIAL_CHARACTERS, "\\\\$1");
+ regex = regex.replace("_", ".").replace("%", ".*"); // transform some special characters to be 'like'.
+
+ this.compiled = Pattern.compile(regex, Pattern.DOTALL);
+ }
+
+ @Override
+ public String toString() {
+ return leftExpr.toString() + " SIMILAR TO '" + pattern + "'";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveConverter.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveConverter.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveConverter.java
index 8429759..2e16813 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveConverter.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveConverter.java
@@ -705,10 +705,7 @@ public class HiveConverter extends HiveParserBaseVisitor<Expr>{
/**
* This method parse operators for equals expressions as follows:
- * =, <>, !=, >=, >, <=, <, IN, NOT IN, LIKE
- *
- * And Tajo doesn't provide some operators as follows:
- * REGEXP, RLIKE
+ * =, <>, !=, >=, >, <=, <, IN, NOT IN, LIKE, REGEXP, RLIKE
*
* In this case, this make RuntimeException>
*
@@ -763,9 +760,9 @@ public class HiveConverter extends HiveParserBaseVisitor<Expr>{
} else if(keyword.equals("!=")) {
type = OpType.NotEquals;
} else if(keyword.equals("REGEXP")) {
- throw new RuntimeException("Unexpected operator : REGEXP");
+ type = OpType.Regexp;
} else if(keyword.equals("RLIKE")) {
- throw new RuntimeException("Unexpected operator : RLIKE");
+ type = OpType.Regexp;
} else if(keyword.equals("LIKE")) {
type = OpType.LikePredicate;
}
@@ -774,8 +771,12 @@ public class HiveConverter extends HiveParserBaseVisitor<Expr>{
if(type != null && right != null) {
if(type.equals(OpType.LikePredicate)) {
- LikePredicate like = new LikePredicate(isNot, (ColumnReferenceExpr)left, (LiteralValue)right);
+ PatternMatchPredicate like = new PatternMatchPredicate(OpType.LikePredicate,
+ isNot, left, right);
current = like;
+ } else if (type.equals(OpType.Regexp)) {
+ PatternMatchPredicate regex = new PatternMatchPredicate(OpType.Regexp, isNot, left, right);
+ current = regex;
} else {
BinaryOperator binaryOperator = new BinaryOperator(type, left, right);
current = binaryOperator;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 97c66b5..e38a3b3 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -21,6 +21,7 @@ package org.apache.tajo.engine.parser;
import com.google.common.base.Preconditions;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.tajo.algebra.*;
import org.apache.tajo.algebra.Aggregation.GroupType;
@@ -623,13 +624,41 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
- public LikePredicate visitLike_predicate(SQLParser.Like_predicateContext ctx) {
- boolean not = ctx.NOT() != null;
-
- ColumnReferenceExpr predicand = (ColumnReferenceExpr) visit(ctx.column_reference());
+ public Expr visitPattern_matching_predicate(SQLParser.Pattern_matching_predicateContext ctx) {
+ Expr predicand = visitChildren(ctx.numeric_primary());
Expr pattern = new LiteralValue(stripQuote(ctx.Character_String_Literal().getText()),
LiteralType.String);
- return new LikePredicate(not, predicand, pattern);
+
+ if (checkIfExist(ctx.pattern_matcher().negativable_matcher())) {
+ boolean not = ctx.pattern_matcher().NOT() != null;
+ Negativable_matcherContext matcher = ctx.pattern_matcher().negativable_matcher();
+ if (checkIfExist(matcher.LIKE())) {
+ return new PatternMatchPredicate(OpType.LikePredicate, not, predicand, pattern);
+ } else if (checkIfExist(matcher.ILIKE())) {
+ return new PatternMatchPredicate(OpType.LikePredicate, not, predicand, pattern, true);
+ } else if (checkIfExist(matcher.SIMILAR())) {
+ return new PatternMatchPredicate(OpType.SimilarToPredicate, not, predicand, pattern);
+ } else if (checkIfExist(matcher.REGEXP()) || checkIfExist(matcher.RLIKE())) {
+ return new PatternMatchPredicate(OpType.Regexp, not, predicand, pattern);
+ } else {
+ throw new SQLSyntaxError("Unsupported predicate: " + matcher.getText());
+ }
+ } else if (checkIfExist(ctx.pattern_matcher().regex_matcher())) {
+ Regex_matcherContext matcher = ctx.pattern_matcher().regex_matcher();
+ if (checkIfExist(matcher.Similar_To())) {
+ return new PatternMatchPredicate(OpType.Regexp, false, predicand, pattern, false);
+ } else if (checkIfExist(matcher.Not_Similar_To())) {
+ return new PatternMatchPredicate(OpType.Regexp, true, predicand, pattern, false);
+ } else if (checkIfExist(matcher.Similar_To_Case_Insensitive())) {
+ return new PatternMatchPredicate(OpType.Regexp, false, predicand, pattern, true);
+ } else if (checkIfExist(matcher.Not_Similar_To_Case_Insensitive())) {
+ return new PatternMatchPredicate(OpType.Regexp, true, predicand, pattern, true);
+ } else {
+ throw new SQLSyntaxError("Unsupported predicate: " + matcher.getText());
+ }
+ } else {
+ throw new SQLSyntaxError("Unsupported predicate: " + ctx.pattern_matcher().getText());
+ }
}
@Override
@@ -639,6 +668,15 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
+ public Expr visitLiteral(@NotNull SQLParser.LiteralContext ctx) {
+ if (checkIfExist(ctx.NULL())) {
+ return new NullValue();
+ } else {
+ return visitChildren(ctx);
+ }
+ }
+
+ @Override
public ColumnReferenceExpr visitColumn_reference(SQLParser.Column_referenceContext ctx) {
ColumnReferenceExpr column = new ColumnReferenceExpr(ctx.name.getText());
if (ctx.tb_name != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 17f3634..6dd3e77 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -33,6 +33,7 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.eval.*;
import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock;
import org.apache.tajo.engine.planner.logical.*;
@@ -918,6 +919,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
switch(expr.getType()) {
// constants
+ case Null:
+ return new ConstEval(NullDatum.get());
+
case Literal:
LiteralValue literal = (LiteralValue) expr;
switch (literal.getValueType()) {
@@ -951,12 +955,27 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
NotExpr notExpr = (NotExpr) expr;
return new NotEval(createEvalTree(plan, block, notExpr.getChild()));
- // binary expressions
+ // pattern matching predicates
case LikePredicate:
- LikePredicate like = (LikePredicate) expr;
- FieldEval field = (FieldEval) createEvalTree(plan, block, like.getColumnRef());
- ConstEval pattern = (ConstEval) createEvalTree(plan, block, like.getPattern());
- return new LikeEval(like.isNot(), field, pattern);
+ case SimilarToPredicate:
+ case Regexp:
+ PatternMatchPredicate patternMatch = (PatternMatchPredicate) expr;
+ EvalNode field = createEvalTree(plan, block, patternMatch.getPredicand());
+ ConstEval pattern = (ConstEval) createEvalTree(plan, block, patternMatch.getPattern());
+
+ // A pattern is a const value in pattern matching predicates.
+ // In a binary expression, the result is always null if a const value in left or right side is null.
+ if (pattern.getValue() instanceof NullDatum) {
+ return new ConstEval(NullDatum.get());
+ } else {
+ if (expr.getType() == OpType.LikePredicate) {
+ return new LikePredicateEval(patternMatch.isNot(), field, pattern, patternMatch.isCaseInsensitive());
+ } else if (expr.getType() == OpType.SimilarToPredicate) {
+ return new SimilarToPredicateEval(patternMatch.isNot(), field, pattern);
+ } else {
+ return new RegexPredicateEval(patternMatch.isNot(), field, pattern, patternMatch.isCaseInsensitive());
+ }
+ }
case InPredicate: {
InPredicate inPredicate = (InPredicate) expr;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
index 4de4d77..b3be308 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
@@ -624,6 +624,7 @@ public class TestEvalTree {
"select name, score, age from people where name like '%bc'", // 0"
"select name, score, age from people where name like 'aa%'", // 1"
"select name, score, age from people where name not like '%bc'", // 2"
+ "select name, score, age from people where name like '.*b_'", // 3"
};
@Test
@@ -631,6 +632,7 @@ public class TestEvalTree {
EvalNode expr;
Schema peopleSchema = cat.getTableDesc("people").getMeta().getSchema();
+ // prefix
expr = getRootSelection(LIKE[0]);
EvalContext evalCtx = expr.newContext();
expr.eval(evalCtx, peopleSchema, tuples[0]);
@@ -640,7 +642,7 @@ public class TestEvalTree {
expr.eval(evalCtx, peopleSchema, tuples[2]);
assertTrue(expr.terminate(evalCtx).asBool());
- // prefix
+ // suffix
expr = getRootSelection(LIKE[1]);
evalCtx = expr.newContext();
expr.eval(evalCtx, peopleSchema, tuples[0]);
@@ -714,7 +716,6 @@ public class TestEvalTree {
private static void assertJsonSerDer(EvalNode expr) {
String json = expr.toJson();
- System.out.println(json);
EvalNode fromJson = CoreGsonHelper.fromJson(json, EvalNode.class);
assertEquals(expr, fromJson);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/ExprTestBase.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/ExprTestBase.java
new file mode 100644
index 0000000..2ba6ee6
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/ExprTestBase.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.function;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.algebra.Expr;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.datum.TextDatum;
+import org.apache.tajo.engine.eval.EvalContext;
+import org.apache.tajo.engine.eval.EvalNode;
+import org.apache.tajo.engine.json.CoreGsonHelper;
+import org.apache.tajo.engine.parser.SQLAnalyzer;
+import org.apache.tajo.engine.planner.LogicalPlan;
+import org.apache.tajo.engine.planner.LogicalPlanner;
+import org.apache.tajo.engine.planner.PlanningException;
+import org.apache.tajo.engine.planner.Target;
+import org.apache.tajo.master.TajoMaster;
+import org.apache.tajo.storage.LazyTuple;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ExprTestBase {
+ private static TajoTestingCluster util;
+ private static CatalogService cat;
+ private static SQLAnalyzer analyzer;
+ private static LogicalPlanner planner;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ util = new TajoTestingCluster();
+ util.startCatalogCluster();
+ cat = util.getMiniCatalogCluster().getCatalog();
+ for (FunctionDesc funcDesc : TajoMaster.initBuiltinFunctions()) {
+ cat.registerFunction(funcDesc);
+ }
+
+ analyzer = new SQLAnalyzer();
+ planner = new LogicalPlanner(cat);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ util.shutdownCatalogCluster();
+ }
+
+ private static void assertJsonSerDer(EvalNode expr) {
+ String json = expr.toJson();
+ EvalNode fromJson = CoreGsonHelper.fromJson(json, EvalNode.class);
+ assertEquals(expr, fromJson);
+ }
+
+ private static Target[] getRawTargets(String query) throws PlanningException {
+ Expr expr = analyzer.parse(query);
+ LogicalPlan plan = planner.createPlan(expr);
+ Target [] targets = plan.getRootBlock().getTargetListManager().getUnresolvedTargets();
+ if (targets == null) {
+ throw new PlanningException("Wrong query statement or query plan: " + query);
+ }
+ for (Target t : targets) {
+ assertJsonSerDer(t.getEvalTree());
+ }
+ return targets;
+ }
+
+ public void testSimpleEval(String query, String [] expected) {
+ testEval(null, null, null, query, expected);
+ }
+
+ public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected) {
+ LazyTuple lazyTuple;
+ VTuple vtuple = null;
+ Schema inputSchema = null;
+ if (schema != null) {
+ inputSchema = (Schema) schema.clone();
+ inputSchema.setQualifier(tableName, true);
+
+ int targetIdx [] = new int[inputSchema.getColumnNum()];
+ for (int i = 0; i < targetIdx.length; i++) {
+ targetIdx[i] = i;
+ }
+
+ lazyTuple = new LazyTuple(inputSchema, Bytes.splitPreserveAllTokens(csvTuple.getBytes(), ',', targetIdx), 0);
+ vtuple = new VTuple(inputSchema.getColumnNum());
+ for (int i = 0; i < inputSchema.getColumnNum(); i++) {
+ // If null value occurs, null datum is manually inserted to an input tuple.
+ if (lazyTuple.get(i) instanceof TextDatum && lazyTuple.getText(i).asChars().equals("")) {
+ vtuple.put(i, NullDatum.get());
+ } else {
+ vtuple.put(i, lazyTuple.get(i));
+ }
+ }
+ cat.addTable(new TableDescImpl(tableName, inputSchema, CatalogProtos.StoreType.CSV, new Options(), new Path("/")));
+ }
+
+ Target [] targets = null;
+
+ try {
+ targets = getRawTargets(query);
+ } catch (PlanningException e) {
+ assertTrue("Wrong query statement: " + query, false);
+ }
+
+ EvalContext [] evalContexts = new EvalContext[targets.length];
+ Tuple outTuple = new VTuple(targets.length);
+ for (int i = 0; i < targets.length; i++) {
+ EvalNode eval = targets[i].getEvalTree();
+ evalContexts[i] = eval.newContext();
+ eval.eval(evalContexts[i], inputSchema, vtuple);
+ outTuple.put(i, eval.terminate(evalContexts[i]));
+ }
+
+ if (schema != null) {
+ cat.deleteTable(tableName);
+ }
+
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals(query, expected[i], outTuple.get(i).asChars());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java
new file mode 100644
index 0000000..2b43795
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.function;
+
+import org.apache.tajo.catalog.Schema;
+import org.junit.Test;
+
+import static org.apache.tajo.common.TajoDataTypes.Type.TEXT;
+
+public class TestPatternMatchingPredicates extends ExprTestBase {
+
+ @Test
+ public void testLike() {
+ Schema schema = new Schema();
+ schema.addColumn("col1", TEXT);
+
+ // test for null values
+ testEval(schema, "table1", ",", "select col1 like 'a%' from table1", new String[]{""});
+ testSimpleEval("select null like 'a%'", new String[]{""});
+
+ testEval(schema, "table1", "abc", "select col1 like '%c' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 like 'a%' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 like '_bc' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 like 'ab_' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 like '_b_' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 like '%b%' from table1", new String[]{"t"});
+
+ // test for escaping regular expressions
+ testEval(schema, "table1", "abc", "select col1 not like '.bc' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 not like '.*bc' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 not like '.bc' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 not like '*bc' from table1", new String[]{"t"});
+
+ // test for case sensitive
+ testEval(schema, "table1", "abc", "select col1 not like '%C' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 not like 'A%' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 not like '_BC' from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc", "select col1 not like '_C_' from table1", new String[]{"t"});
+ }
+
+ @Test
+ public void testILike() {
+ testSimpleEval("select 'abc' ilike '%c'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike 'a%'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike '_bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike 'ab_'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike '_b_'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike '%b%'", new String[]{"t"});
+
+ // test for escaping regular expressions
+ testSimpleEval("select 'abc' not like '.bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' not like '.*bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' not like '.bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' not like '*bc'", new String[]{"t"});
+
+ // test for case insensitive
+ testSimpleEval("select 'abc' ilike '%C'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike 'A%'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike '_BC'", new String[]{"t"});
+ testSimpleEval("select 'abc' ilike '_B_'", new String[]{"t"});
+ }
+
+ @Test
+ public void testSimilarToLike() {
+ testSimpleEval("select 'abc' similar to '%c'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to 'a%'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to '_bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to 'ab_'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to '_b_'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to '%b%'", new String[]{"t"});
+
+ // test for now allowed
+ testSimpleEval("select 'abc' not similar to '.bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' not similar to 'ab.'", new String[]{"t"});
+
+ // test for escaping regular expressions
+ testSimpleEval("select 'abc' similar to '(a|f)b%'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to '[a-z]b%'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to '_+bc'", new String[]{"t"});
+ testSimpleEval("select 'abc' similar to 'abc'", new String[]{"t"});
+
+ // test for case sensitive
+ testSimpleEval("select 'abc' not similar to '%C'", new String[]{"t"});
+ testSimpleEval("select 'abc' not similar to '_Bc'", new String[]{"t"});
+ }
+
+ @Test
+ public void testRegexWithSimilarOperator() {
+ testSimpleEval("select 'abc' ~ '.*c'", new String[]{"t"});
+ testSimpleEval("select 'abc' ~ '.*c$'", new String[]{"t"});
+ testSimpleEval("select 'aaabc' ~ '([a-z]){3}bc'", new String[]{"t"});
+
+ // for negative condition
+ testSimpleEval("select 'abc' !~ '.*c$'", new String[]{"f"});
+
+ // for case sensitivity
+ testSimpleEval("select 'abc' ~ '.*C'", new String[]{"f"});
+
+ // for case insensitivity
+ testSimpleEval("select 'abc' ~* '.*C'", new String[]{"t"});
+ testSimpleEval("select 'abc' !~* '.*C'", new String[]{"f"});
+ }
+
+ @Test
+ public void testRegexp() {
+ testSimpleEval("select 'abc' regexp '.*c'", new String[]{"t"});
+ testSimpleEval("select 'abc' regexp '.*c$'", new String[]{"t"});
+
+ // for negative condition
+ testSimpleEval("select 'abc' not regexp '.*c$'", new String[]{"f"});
+ }
+
+ @Test
+ public void testRLike() {
+ testSimpleEval("select 'abc' rlike '.*c'", new String[]{"t"});
+ testSimpleEval("select 'abc' rlike '.*c$'", new String[]{"t"});
+
+ // for negative condition
+ testSimpleEval("select 'abc' not rlike '.*c$'", new String[]{"f"});
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
index e9a0c65..dbd3bde 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
@@ -90,27 +90,31 @@ public class TestInsertQuery {
ResultSet res = tpch.execute("select * from " + tableName);
assertTrue(res.next());
assertEquals(1, res.getLong(1));
- assertEquals("null", res.getString(2));
+ assertTrue(0f == res.getFloat(2));
+ assertTrue(res.wasNull());
assertTrue(17.0 == res.getFloat(3));
assertTrue(res.next());
assertEquals(1, res.getLong(1));
- assertEquals("null", res.getString(2));
+ assertTrue(0f == res.getFloat(2));
+ assertTrue(res.wasNull());
assertTrue(36.0 == res.getFloat(3));
assertTrue(res.next());
assertEquals(2, res.getLong(1));
- assertEquals("null", res.getString(2));
+ assertTrue(0f == res.getFloat(2));
+ assertTrue(res.wasNull());
assertTrue(38.0 == res.getFloat(3));
assertTrue(res.next());
- assertEquals(3, res.getLong(1));
- assertEquals("null", res.getString(2));
+ assertTrue(0f == res.getFloat(2));
+ assertTrue(res.wasNull());
assertTrue(45.0 == res.getFloat(3));
assertTrue(res.next());
assertEquals(3, res.getLong(1));
- assertEquals("null", res.getString(2));
+ assertTrue(0f == res.getFloat(2));
+ assertTrue(res.wasNull());
assertTrue(49.0 == res.getFloat(3));
assertFalse(res.next());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index df6eaa0..270f915 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -279,15 +279,15 @@ public class TestSelectQuery {
"end as cond from region");
try {
- Map<Integer, String> result = Maps.newHashMap();
- result.put(0, "null");
- result.put(1, "11");
- result.put(2, "12");
- result.put(3, "13");
- result.put(4, "14");
+ Map<Integer, Integer> result = Maps.newHashMap();
+ result.put(0, 0);
+ result.put(1, 11);
+ result.put(2, 12);
+ result.put(3, 13);
+ result.put(4, 14);
int cnt = 0;
while(res.next()) {
- assertEquals(result.get(res.getInt(1)), res.getString(2));
+ assertTrue(result.get(res.getInt(1)) == res.getInt(2));
cnt++;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7b0dec6a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
index c10fa50..c2b511c 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
@@ -113,8 +113,6 @@ public class LazyTuple implements Tuple {
else if (textBytes.length > fieldId && (textBytes[fieldId] != null)) {
values[fieldId] = createByTextBytes(schema.getColumn(fieldId).getDataType().getType(), textBytes[fieldId]);
textBytes[fieldId] = null;
- } else {
- //values[fieldId] = NullDatum.get();
}
return values[fieldId];
}