You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2013/09/27 11:26:09 UTC
git commit: TAJO-210: Implement String concatenation operator.
(hyunsik)
Updated Branches:
refs/heads/master 3d30e2532 -> f4600ddfa
TAJO-210: Implement String concatenation operator. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/f4600ddf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/f4600ddf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/f4600ddf
Branch: refs/heads/master
Commit: f4600ddfad489b16c6b584997be6edf7cb01216d
Parents: 3d30e25
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Sep 27 17:57:09 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Sep 27 18:25:33 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 4 +-
.../java/org/apache/tajo/algebra/OpType.java | 3 +-
.../org/apache/tajo/engine/parser/SQLLexer.g4 | 2 +
.../org/apache/tajo/engine/parser/SQLParser.g4 | 11 +++--
.../tajo/engine/eval/BasicEvalNodeVisitor.java | 2 +-
.../org/apache/tajo/engine/eval/BinaryEval.java | 17 ++++---
.../org/apache/tajo/engine/eval/EvalType.java | 39 +++++++++------
.../tajo/engine/eval/RegexPredicateEval.java | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 29 +++++++++--
.../tajo/engine/planner/LogicalPlanner.java | 2 +
.../master/querymaster/QueryInProgress.java | 3 +-
.../TestStringOperatorsAndFunctions.java | 51 ++++++++++++++++++++
12 files changed, 133 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b981279..7b768f2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,8 +4,10 @@ Release 0.2.0 - unreleased
NEW FEATURES
+ TAJO-206: Implement String concatenation operator (||). (hyunsik)
+
TAJO-213: NULL characters in meta of csv table should be supported.
- (jinho)
+ (jinho)
TAJO-185: Implement split_part function. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
index 9b03fc0..ab4e403 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
@@ -67,10 +67,11 @@ public enum OpType {
ValueList(ValueListExpr.class),
Is,
- // pattern matching predicates
+ // string operator or pattern matching predicates
LikePredicate(PatternMatchPredicate.class),
SimilarToPredicate(PatternMatchPredicate.class),
Regexp(PatternMatchPredicate.class),
+ Concatenate(BinaryOperator.class),
// arithmetic operators
Plus(BinaryOperator.class),
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
index 4a32cf2..46384c8 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
@@ -309,6 +309,7 @@ ASSIGN : ':=';
EQUAL : '=';
SEMI_COLON : ';';
COMMA : ',';
+CONCATENATION_OPERATOR : VERTICAL_BAR VERTICAL_BAR;
NOT_EQUAL : '<>' | '!=' | '~='| '^=' ;
LTH : '<' ;
LEQ : '<=';
@@ -323,6 +324,7 @@ DIVIDE : '/';
MODULAR : '%';
DOT : '.';
UNDERLINE : '_';
+VERTICAL_BAR : '|';
NUMBER : Digit+;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 049d447..2c52b70 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -779,7 +779,11 @@ numeric_value_expression
;
term
- : left=numeric_primary ((MULTIPLY|DIVIDE|MODULAR) right=numeric_primary)*
+ : left=concatenatable_term ((MULTIPLY|DIVIDE|MODULAR) right=concatenatable_term)*
+ ;
+
+concatenatable_term
+ : left=numeric_primary (CONCATENATION_OPERATOR right=numeric_primary)*
;
array
@@ -796,15 +800,16 @@ numeric_primary
;
literal
- : string_value_expr
+ : string_value_expression
| signed_numerical_literal
| NULL
;
-string_value_expr
+string_value_expression
: Character_String_Literal
;
+
signed_numerical_literal
: sign? unsigned_numerical_literal
;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
index 7e34c5a..205e452 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java
@@ -105,7 +105,7 @@ public class BasicEvalNodeVisitor<CONTEXT, RESULT> implements EvalNodeVisitor2<C
case SIMILAR_TO:
result = visitSimilarTo(context, stack, (SimilarToPredicateEval) evalNode);
break;
- case Regex:
+ case REGEX:
result = visitRegex(context, stack, (RegexPredicateEval) evalNode);
break;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java
index 6f742b6..93d3660 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java
@@ -54,19 +54,20 @@ public class BinaryEval extends EvalNode implements Cloneable {
type == EvalType.LTH ||
type == EvalType.GTH ||
type == EvalType.LEQ ||
- type == EvalType.GEQ
- ) {
+ type == EvalType.GEQ ) {
this.returnType = CatalogUtil.newDataTypesWithoutLen(TajoDataTypes.Type.BOOLEAN);
} else if (
type == EvalType.PLUS ||
type == EvalType.MINUS ||
type == EvalType.MULTIPLY ||
type == EvalType.DIVIDE ||
- type == EvalType.MODULAR
- ) {
+ type == EvalType.MODULAR ) {
this.returnType = SchemaUtil.newNoNameSchema(determineType(left.getValueType()[0],
right.getValueType()[0]));
- }
+
+ } else if (type == EvalType.CONCATENATE) {
+ this.returnType = CatalogUtil.newDataTypesWithoutLen(TajoDataTypes.Type.TEXT);
+ }
}
public BinaryEval(PartialBinaryExpr expr) {
@@ -178,6 +179,10 @@ public class BinaryEval extends EvalNode implements Cloneable {
return leftExpr.terminate(binCtx.left).divide(rightExpr.terminate(binCtx.right));
case MODULAR:
return leftExpr.terminate(binCtx.left).modular(rightExpr.terminate(binCtx.right));
+
+ case CONCATENATE:
+ return DatumFactory.createText(leftExpr.terminate(binCtx.left).asChars()
+ + rightExpr.terminate(binCtx.right).asChars());
default:
throw new InvalidEvalException("We does not support " + type + " expression yet");
}
@@ -197,7 +202,7 @@ public class BinaryEval extends EvalNode implements Cloneable {
}
public String toString() {
- return leftExpr +" "+type+" "+rightExpr;
+ return leftExpr +" " + type.getOperatorName() + " "+rightExpr;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
index e10017e..f5a9e56 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/EvalType.java
@@ -20,32 +20,33 @@ package org.apache.tajo.engine.eval;
public enum EvalType {
// Unary expression
- NOT(NotEval.class),
+ NOT(NotEval.class, "!"),
// Binary expression
AND(BinaryEval.class),
OR(BinaryEval.class),
- EQUAL(BinaryEval.class),
+ EQUAL(BinaryEval.class, "="),
IS_NULL(IsNullEval.class),
- NOT_EQUAL(BinaryEval.class),
- LTH(BinaryEval.class),
- LEQ(BinaryEval.class),
- GTH(BinaryEval.class),
- GEQ(BinaryEval.class),
- PLUS(BinaryEval.class),
- MINUS(BinaryEval.class),
- MODULAR(BinaryEval.class),
- MULTIPLY(BinaryEval.class),
- DIVIDE(BinaryEval.class),
+ NOT_EQUAL(BinaryEval.class, "<>"),
+ LTH(BinaryEval.class, "<"),
+ LEQ(BinaryEval.class, "<="),
+ GTH(BinaryEval.class, ">"),
+ GEQ(BinaryEval.class, ">="),
+ PLUS(BinaryEval.class, "+"),
+ MINUS(BinaryEval.class, "-"),
+ MODULAR(BinaryEval.class, "%"),
+ MULTIPLY(BinaryEval.class, "*"),
+ DIVIDE(BinaryEval.class, "/"),
// Function
AGG_FUNCTION(AggFuncCallEval.class),
FUNCTION(FuncCallEval.class),
- // String pattern matching
+ // String operator or pattern matching predicates
LIKE(LikePredicateEval.class),
SIMILAR_TO(SimilarToPredicateEval.class),
- Regex(RegexPredicateEval.class),
+ REGEX(RegexPredicateEval.class),
+ CONCATENATE(BinaryEval.class, "||"),
// Other predicates
CASE(CaseWhenEval.class),
@@ -58,11 +59,21 @@ public enum EvalType {
ROW_CONSTANT(RowConstantEval.class);
private Class<? extends EvalNode> baseClass;
+ private String operatorName;
EvalType(Class<? extends EvalNode> type) {
this.baseClass = type;
}
+ EvalType(Class<? extends EvalNode> type, String text) {
+ this(type);
+ this.operatorName = text;
+ }
+
+ public String getOperatorName() {
+ return operatorName != null ? operatorName : name();
+ }
+
public Class<? extends EvalNode> getBaseClass() {
return this.baseClass;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
index a173d8f..f1e0241 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java
@@ -26,7 +26,7 @@ import java.util.regex.PatternSyntaxException;
public class RegexPredicateEval extends PatternMatchPredicateEval {
@Expose private String operator;
public RegexPredicateEval(boolean not, EvalNode field, ConstEval pattern, boolean caseInsensitive) {
- super(EvalType.Regex, not, field, pattern, caseInsensitive);
+ super(EvalType.REGEX, not, field, pattern, caseInsensitive);
StringBuilder sb = new StringBuilder();
if (not) {
sb.append("!");
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index ae6d38e..b173037 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -543,14 +543,14 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
@Override
public Expr visitTerm(SQLParser.TermContext ctx) {
- Expr current = visitNumeric_primary(ctx.numeric_primary(0));
+ Expr current = visitConcatenatable_term(ctx.concatenatable_term(0));
Expr left;
Expr right;
for (int i = 1; i < ctx.getChildCount(); i++) {
left = current;
TerminalNode operator = (TerminalNode) ctx.getChild(i++);
- right = visitNumeric_primary((Numeric_primaryContext) ctx.getChild(i));
+ right = visitConcatenatable_term((Concatenatable_termContext) ctx.getChild(i));
if (operator.getSymbol().getType() == MULTIPLY) {
current = new BinaryOperator(OpType.Multiply, left, right);
@@ -564,6 +564,27 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
return current;
}
+ @Override public Expr visitConcatenatable_term(SQLParser.Concatenatable_termContext ctx) {
+ Expr current = visitNumeric_primary(ctx.numeric_primary(0));
+
+ Expr left;
+ Expr right;
+ for (int i = 1; i < ctx.getChildCount(); i++) {
+ left = current;
+ i++; // skip '||' operator
+ right = visitNumeric_primary((Numeric_primaryContext) ctx.getChild(i));
+
+ if (left.getType() == OpType.Literal && right.getType() == OpType.Literal) {
+ current = new LiteralValue(((LiteralValue)left).getValue() + ((LiteralValue)right).getValue(),
+ LiteralType.String);
+ } else {
+ current = new BinaryOperator(OpType.Concatenate, left, right);
+ }
+ }
+
+ return current;
+ }
+
@Override
public Expr visitNumeric_primary(SQLParser.Numeric_primaryContext ctx) {
if (ctx.numeric_value_expression() != null) {
@@ -745,8 +766,8 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
- public LiteralValue visitString_value_expr(SQLParser.String_value_exprContext ctx) {
- return new LiteralValue(stripQuote(ctx.getText()), LiteralType.String);
+ public LiteralValue visitString_value_expression(@NotNull SQLParser.String_value_expressionContext ctx) {
+ return new LiteralValue(stripQuote(ctx.Character_String_Literal().getText()), LiteralType.String);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 52f0547..3edb712 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -1002,6 +1002,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
case Multiply:
case Divide:
case Modular:
+ case Concatenate:
BinaryOperator bin = (BinaryOperator) expr;
return new BinaryEval(exprTypeToEvalType(expr.getType()),
createEvalTree(plan, block, bin.getLeft()),
@@ -1125,6 +1126,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
case Multiply: return EvalType.MULTIPLY;
case Divide: return EvalType.DIVIDE;
case Modular: return EvalType.MODULAR;
+ case Concatenate: return EvalType.CONCATENATE;
case Column: return EvalType.FIELD;
case Function: return EvalType.FUNCTION;
default: throw new RuntimeException("Unsupported type: " + type);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
index 15d5b9b..6ce0fed 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
@@ -118,13 +118,14 @@ public class QueryInProgress extends CompositeService {
} catch (InterruptedException e) {
break;
}
- if(System.currentTimeMillis() - startTime > 300 * 1000) {
+ if(System.currentTimeMillis() - startTime > 60 * 1000) {
LOG.warn("Failed to stop QueryMaster:" + queryId);
break;
}
}
super.stop();
+
if(queryMasterRpc != null) {
//TODO release to connection pool
queryMasterRpc.close();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f4600ddf/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
new file mode 100644
index 0000000..9c2a68c
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.function;
+
+
+import org.apache.tajo.catalog.Schema;
+import org.junit.Test;
+
+import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8;
+import static org.apache.tajo.common.TajoDataTypes.Type.INT4;
+import static org.apache.tajo.common.TajoDataTypes.Type.TEXT;
+
+public class TestStringOperatorsAndFunctions extends ExprTestBase {
+
+ @Test
+ public void testConcatenateOnLiteral() {
+ testSimpleEval("select ('abc' || 'def') col1 ", new String[]{"abcdef"});
+ testSimpleEval("select 'abc' || 'def' as col1 ", new String[]{"abcdef"});
+ testSimpleEval("select 1 || 'def' as col1 ", new String[]{"1def"});
+ testSimpleEval("select 'abc' || 2 as col1 ", new String[]{"abc2"});
+ }
+
+ @Test
+ public void testConcatenateOnExpressions() {
+ Schema schema = new Schema();
+ schema.addColumn("col1", TEXT);
+ schema.addColumn("col2", INT4);
+ schema.addColumn("col3", FLOAT8);
+
+ testSimpleEval("select (1+3) || 2 as col1 ", new String[]{"42"});
+
+ testEval(schema, "table1", "abc,2,3.14", "select col1 || col2 || col3 from table1", new String[]{"abc23.14"});
+ testEval(schema, "table1", "abc,2,3.14", "select col1 || '---' || col3 from table1", new String[]{"abc---3.14"});
+ }
+}