You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2013/12/31 09:50:53 UTC
[01/10] git commit: DAG-execplan
Updated Branches:
refs/heads/DAG-execplan 6b9061432 -> f932f5d06
DAG-execplan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/77660afb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/77660afb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/77660afb
Branch: refs/heads/DAG-execplan
Commit: 77660afb5561c9abdd09956128b356a0635b251e
Parents: 6b90614
Author: Jihoon Son <ji...@apache.org>
Authored: Sat Dec 28 15:54:25 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Sat Dec 28 15:54:25 2013 +0900
----------------------------------------------------------------------
.../org/apache/tajo/engine/planner/global/ExecutionPlan.java | 6 ++++++
.../java/org/apache/tajo/engine/planner/logical/ScanNode.java | 1 +
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/77660afb/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
index e91e460..80ea564 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
@@ -432,6 +432,12 @@ public class ExecutionPlan implements GsonObject {
if (plan.inputContext == null) {
plan.inputContext = new InputContext();
}
+ // TODO: update scan and the plan graph
+ for (ScanNode eachScan : plan.inputContext.getScanNodes()) {
+ if (eachScan.equals(node)) {
+ return;
+ }
+ }
plan.inputContext.addScanNode(node);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/77660afb/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java
index bfd25cc..1c6efc8 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java
@@ -145,6 +145,7 @@ public class ScanNode extends RelationNode implements Projectable {
eq = eq && TUtil.checkEquals(this.tableDesc, other.tableDesc);
eq = eq && TUtil.checkEquals(this.qual, other.qual);
eq = eq && TUtil.checkEquals(this.targets, other.targets);
+ eq = eq && TUtil.checkEquals(this.renamedSchema, other.renamedSchema);
return eq;
}
[02/10] git commit: TAJO-381: Implement find_in_set function. (Jae
Young Lee via hyunsik)
Posted by ji...@apache.org.
TAJO-381: Implement find_in_set function. (Jae Young Lee via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/4c757900
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/4c757900
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/4c757900
Branch: refs/heads/DAG-execplan
Commit: 4c7579009c137570ecc412d1365ace77fbbd4b87
Parents: 319a377
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun Dec 29 01:12:18 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun Dec 29 01:13:01 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tajo/engine/function/string/FindInSet.java | 104 +++++++++++++++++++
.../java/org/apache/tajo/master/TajoMaster.java | 6 ++
.../apache/tajo/engine/eval/ExprTestBase.java | 14 ++-
.../TestStringOperatorsAndFunctions.java | 29 +++++-
5 files changed, 150 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4c757900/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 858b185..ff45694 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@ Release 0.8.0 - unreleased
NEW FEATURES
+ TAJO-381: Implement find_in_set function. (Jae Young Lee via hyunsik)
+
TAJO-439: Time literal support. (DaeMyung Kang via jihoon)
TAJO-437: Timestamp literal support. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4c757900/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java
new file mode 100644
index 0000000..17df825
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.function.string;
+
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.engine.function.GeneralFunction;
+import org.apache.tajo.storage.Tuple;
+
+/**
+ * find_in_set(text,str_array) - Returns the first occurrence of str in str_array where str_array
+ * is a comma-delimited string.
+ *
+ * Returns null if either argument is null.
+ * Returns 0 if the first argument has any commas.
+ *
+ * Example:
+ * SELECT find_in_set('cr','crt,c,cr,c,def') FROM src LIMIT 1;\n"
+ * -> result: 3
+ */
+public class FindInSet extends GeneralFunction {
+ public FindInSet() {
+ super(new Column[]{
+ new Column("text", TajoDataTypes.Type.TEXT),
+ new Column("str_array", TajoDataTypes.Type.TEXT)
+ });
+ }
+
+ @Override
+ public Datum eval(Tuple params) {
+ Datum finding = params.get(0);
+ Datum textArray = params.get(1);
+
+ if (finding instanceof NullDatum || textArray instanceof NullDatum) {
+ return NullDatum.get();
+ }
+
+ byte[] searchBytes = finding.asByteArray();
+
+ // Returns 0 if the first argument has any commas.
+ for (int i = 0; i < finding.size(); i++) {
+ if (searchBytes[i] == ',') {
+ return DatumFactory.createInt4(0);
+ }
+ }
+
+ byte[] arrayData = textArray.asByteArray();
+ int findingLength = finding.size();
+
+ int posInTextArray = 0;
+ int curLengthOfCandidate = 0;
+ boolean matching = true;
+
+ for (int i = 0; i < textArray.size(); i++) {
+
+ if (arrayData[i] == ',') {
+ posInTextArray++;
+ if (matching && curLengthOfCandidate == findingLength) {
+ return DatumFactory.createInt4(posInTextArray);
+ } else {
+ matching = true;
+ curLengthOfCandidate = 0;
+ }
+ } else {
+ if (curLengthOfCandidate + 1 <= findingLength) {
+ if (!matching || searchBytes[curLengthOfCandidate] != arrayData[i]) {
+ matching = false;
+ }
+ } else {
+ matching = false;
+ }
+ curLengthOfCandidate++;
+ }
+
+ }
+
+ if (matching && curLengthOfCandidate == findingLength) {
+ posInTextArray++;
+ return DatumFactory.createInt4(posInTextArray);
+ } else {
+ return DatumFactory.createInt4(0);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4c757900/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMaster.java
index 40277cb..0093ff7 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -834,6 +834,12 @@ public class TajoMaster extends CompositeService {
new FunctionDesc("decode", Decode.class, FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.TEXT),
CatalogUtil.newSimpleDataTypeArray(Type.TEXT, Type.TEXT)));
+
+ sqlFuncs.add(
+ new FunctionDesc("find_in_set", FindInSet.class, FunctionType.GENERAL,
+ CatalogUtil.newSimpleDataType(Type.INT4),
+ CatalogUtil.newSimpleDataTypeArray(Type.TEXT, Type.TEXT)));
+
return sqlFuncs;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4c757900/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 3c150ec..092a5bc 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -91,7 +91,13 @@ public class ExprTestBase {
testEval(null, null, null, query, expected);
}
- public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected) throws IOException {
+ public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected)
+ throws IOException {
+ testEval(schema, tableName, csvTuple, query, expected, ',');
+ }
+
+ public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected,
+ char delimiter) throws IOException {
LazyTuple lazyTuple;
VTuple vtuple = null;
Schema inputSchema = null;
@@ -104,7 +110,8 @@ public class ExprTestBase {
targetIdx[i] = i;
}
- lazyTuple = new LazyTuple(inputSchema, Bytes.splitPreserveAllTokens(csvTuple.getBytes(), ',', targetIdx), 0);
+ lazyTuple =
+ new LazyTuple(inputSchema, Bytes.splitPreserveAllTokens(csvTuple.getBytes(), delimiter, targetIdx),0);
vtuple = new VTuple(inputSchema.getColumnNum());
for (int i = 0; i < inputSchema.getColumnNum(); i++) {
// If null value occurs, null datum is manually inserted to an input tuple.
@@ -114,7 +121,8 @@ public class ExprTestBase {
vtuple.put(i, lazyTuple.get(i));
}
}
- cat.addTable(new TableDesc(tableName, inputSchema, CatalogProtos.StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()));
+ cat.addTable(new TableDesc(tableName, inputSchema, CatalogProtos.StoreType.CSV, new Options(),
+ CommonTestingUtil.getTestDir()));
}
Target [] targets = null;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4c757900/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
index ac350fc..0df05b7 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
@@ -539,7 +539,8 @@ public class TestStringOperatorsAndFunctions extends ExprTestBase {
@Test
public void testEncode() throws IOException {
testSimpleEval("select encode('Hello\nworld', 'base64') ", new String[]{"SGVsbG8Kd29ybGQ="});
- testSimpleEval("select encode('Hello\nworld', 'hex') ", new String[]{"0x480x650x6c0x6c0x6f0x0a0x770x6f0x720x6c0x64"});
+ testSimpleEval("select encode('Hello\nworld', 'hex') ",
+ new String[]{"0x480x650x6c0x6c0x6f0x0a0x770x6f0x720x6c0x64"});
testSimpleEval("select encode('한글', 'base64') ", new String[]{"7ZWc6riA"});
testSimpleEval("select encode('한글', 'hex') ", new String[]{"0xd55c0xae00"});
testSimpleEval("select encode('한글\n테스트\t입니다.', 'hex') ",
@@ -549,7 +550,8 @@ public class TestStringOperatorsAndFunctions extends ExprTestBase {
@Test
public void testDecode() throws IOException {
- testSimpleEval("select decode('SGVsbG8Kd29ybGQ=', 'base64') ", new String[]{StringEscapeUtils.escapeJava("Hello\nworld")});
+ testSimpleEval("select decode('SGVsbG8Kd29ybGQ=', 'base64') ",
+ new String[]{StringEscapeUtils.escapeJava("Hello\nworld")});
testSimpleEval("select decode('0x480x650x6c0x6c0x6f0x0a0x770x6f0x720x6c0x64', 'hex') ",
new String[]{StringEscapeUtils.escapeJava("Hello\nworld")});
testSimpleEval("select decode('7ZWc6riA', 'base64') ", new String[]{StringEscapeUtils.escapeJava("한글")});
@@ -557,4 +559,27 @@ public class TestStringOperatorsAndFunctions extends ExprTestBase {
testSimpleEval("select decode('0xd55c0xae000x0a0xd14c0xc2a40xd2b80x090xc7850xb2c80xb2e40x2e', 'hex') ",
new String[]{StringEscapeUtils.escapeJava("한글\n" + "테스트\t입니다.")});
}
+
+ @Test
+ public void testFindInSet() throws IOException {
+ // abnormal cases
+ testSimpleEval("select find_in_set('cr','crt') as col1 ", new String[]{"0"}); // there is no matched string
+ testSimpleEval("select find_in_set('c,r','crt,c,cr,c,def') as col1 ", new String[]{"0"}); // abnormal parameter
+
+ // normal cases
+ testSimpleEval("select find_in_set('crt','crt,c,cr,d,def') as col1 ", new String[]{"1"});
+ testSimpleEval("select find_in_set('c','crt,c,cr,d,def') as col1 ", new String[]{"2"});
+ testSimpleEval("select find_in_set('def','crt,c,cr,d,def') as col1 ", new String[]{"5"});
+ // unicode test
+ testSimpleEval("select find_in_set('딸기','사과,배,옥수수,감자,딸기,수박') as col1 ", new String[]{"5"});
+
+ // null test
+ Schema schema = new Schema();
+ schema.addColumn("col1", TEXT);
+ schema.addColumn("col2", TEXT);
+ testEval(schema, "table1", "|crt,c,cr,c,def", "select find_in_set(col1, col2) is null from table1",
+ new String[]{"t"}, '|');
+ testEval(schema, "table1", "cr|", "select find_in_set(col1, col2) is null from table1",
+ new String[]{"t"}, '|');
+ }
}
[09/10] git commit: DAG-execplan
Posted by ji...@apache.org.
DAG-execplan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/34456a45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/34456a45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/34456a45
Branch: refs/heads/DAG-execplan
Commit: 34456a4505859eb4e4f72523dc9cb8058c3fa0ff
Parents: 9190c2d
Author: Jihoon Son <ji...@apache.org>
Authored: Mon Dec 30 18:49:08 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Mon Dec 30 18:49:08 2013 +0900
----------------------------------------------------------------------
.../engine/planner/PhysicalPlannerImpl.java | 2 +-
.../apache/tajo/engine/planner/PlannerUtil.java | 20 ++++++++++----------
.../engine/planner/global/ExecutionPlan.java | 7 ++++---
.../engine/planner/global/GlobalPlanner.java | 10 +++++-----
.../planner/logical/TableSubQueryNode.java | 4 ++++
5 files changed, 24 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/34456a45/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 0103975..1e4fba9 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -197,7 +197,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
case TABLE_SUBQUERY: {
TableSubQueryNode subQueryNode = (TableSubQueryNode) logicalNode;
- leftExec = createPlanRecursive(ctx, plan, subQueryNode.getSubQuery());
+ leftExec = createPlanRecursive(ctx, plan, plan.getChild(subQueryNode, 0));
if (plan.getParentCount(logicalNode) > 1) {
return new MultiOutputExec(ctx, leftExec.getSchema(), leftExec, plan.getParentCount(logicalNode));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/34456a45/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
index 5164b65..5936253 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
@@ -492,18 +492,18 @@ public class PlannerUtil {
Preconditions.checkNotNull(executionPlan);
Preconditions.checkNotNull(type);
- if (executionPlan.hasPlanGroup()) {
- return findTopNode(executionPlan.getFirstPlanGroup().toLinkedLogicalNode(), type);
- } else {
- return null;
- }
-// LogicalNodeFinderForExecPlan finder = new LogicalNodeFinderForExecPlan(type, executionPlan);
-// finder.find();
-//
-// if (finder.getFoundNodes().size() == 0) {
+// if (executionPlan.hasPlanGroup()) {
+// return findTopNode(executionPlan.getFirstPlanGroup().toLinkedLogicalNode(), type);
+// } else {
// return null;
// }
-// return (T) finder.getFoundNodes().get(0);
+ LogicalNodeFinderForExecPlan finder = new LogicalNodeFinderForExecPlan(type, executionPlan);
+ finder.find();
+
+ if (finder.getFoundNodes().size() == 0) {
+ return null;
+ }
+ return (T) finder.getFoundNodes().get(0);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/34456a45/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
index a6c3bcd..84780c6 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
@@ -495,7 +495,7 @@ public class ExecutionPlan implements GsonObject {
if (node.getChild() != null) {
LogicalNode child = node.getChild();
plan.add(child, node, edgeType);
-// node.setChild(null);
+ node.setChild(null);
visit(child, edgeType);
}
}
@@ -512,13 +512,13 @@ public class ExecutionPlan implements GsonObject {
if (node.getLeftChild() != null) {
child = node.getLeftChild();
plan.add(child, node, EdgeType.LEFT);
-// node.setLeftChild(null);
+ node.setLeftChild(null);
visit(child, EdgeType.LEFT);
}
if (node.getRightChild() != null) {
child = node.getRightChild();
plan.add(child, node, EdgeType.RIGHT);
-// node.setRightChild(null);
+ node.setRightChild(null);
visit(child, EdgeType.RIGHT);
}
}
@@ -526,6 +526,7 @@ public class ExecutionPlan implements GsonObject {
private void visitTableSubQuery(TableSubQueryNode node, EdgeType edgeType) throws PlanningException {
LogicalNode child = node.getSubQuery();
plan.add(child, node, edgeType);
+ node.nullifySubQuery();
visit(child, edgeType);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/34456a45/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index b9206a1..fe7d177 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -579,11 +579,11 @@ public class GlobalPlanner {
private LogicalNode handleUnaryNode(GlobalPlanContext context, LogicalNode child, LogicalNode node) {
ExecutionBlock execBlock = context.execBlockMap.remove(child.getPID());
-// if (node instanceof UnaryNode) {
-// ((UnaryNode)node).setChild(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
-// } else if (node instanceof TableSubQueryNode) {
-// ((TableSubQueryNode) node).setSubQuery(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
-// }
+ if (node instanceof UnaryNode) {
+ ((UnaryNode)node).setChild(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
+ } else if (node instanceof TableSubQueryNode) {
+ ((TableSubQueryNode) node).setSubQuery(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
+ }
execBlock.setPlan(node);
context.execBlockMap.put(node.getPID(), execBlock);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/34456a45/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
index 335d12f..49a4b68 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
@@ -58,6 +58,10 @@ public class TableSubQueryNode extends RelationNode implements Projectable {
setInSchema(subQuery.getInSchema());
}
+ public void nullifySubQuery() {
+ this.subQuery = null;
+ }
+
public LogicalNode getSubQuery() {
return subQuery;
}
[10/10] git commit: DAG-execplan: add a test case,
but more tests are required
Posted by ji...@apache.org.
DAG-execplan: add a test case, but more tests are required
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/f932f5d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/f932f5d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/f932f5d0
Branch: refs/heads/DAG-execplan
Commit: f932f5d064b6814ed4ada2f978d376ff618537a3
Parents: 34456a4
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Dec 31 17:49:35 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Dec 31 17:49:35 2013 +0900
----------------------------------------------------------------------
.../engine/planner/global/ExecutionBlock.java | 6 +++-
.../engine/planner/global/ExecutionPlan.java | 23 ++++++++++-----
.../engine/planner/global/GlobalPlanner.java | 10 +++----
.../planner/logical/TableSubQueryNode.java | 14 ++++++++-
.../planner/global/TestExecutionPlan.java | 30 ++++++++++++++++++++
.../tajo/engine/query/TestCaseByCases.java | 28 ++++++++++++++++++
.../src/test/queries/tajo266_case.sql | 9 ++++++
7 files changed, 106 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java
index 378726b..250b842 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java
@@ -54,7 +54,11 @@ public class ExecutionBlock {
}
public void setPlan(LogicalNode plan) {
- executionPlan.setPlan(plan);
+ try {
+ executionPlan.setPlan((LogicalNode) plan.clone());
+ } catch (CloneNotSupportedException e) {
+
+ }
}
public void setPlan(ExecutionPlan plan) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
index 84780c6..f199be7 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
@@ -88,20 +88,29 @@ public class ExecutionPlan implements GsonObject {
public LogicalNode toLinkedLogicalNode() {
LogicalNode[] nodes = this.nodes.toArray(new LogicalNode[this.nodes.size()]);
- for (int i = 0; i < nodes.length;) {
+ toLinkedLogicalNode(nodes, 0);
+ return nodes[0];
+ }
+
+ private int toLinkedLogicalNode(LogicalNode[] nodes, int i) {
+ if (i < nodes.length) {
if (nodes[i] instanceof UnaryNode) {
- ((UnaryNode)nodes[i]).setChild(nodes[++i]);
+ ((UnaryNode)nodes[i]).setChild(nodes[i+1]);
+ return toLinkedLogicalNode(nodes, i + 1);
} else if (nodes[i] instanceof BinaryNode) {
((BinaryNode)nodes[i]).setLeftChild(nodes[i+1]);
- ((BinaryNode)nodes[i]).setRightChild(nodes[i+2]);
- i += 2;
+ int rightChildIndex = toLinkedLogicalNode(nodes, i+1);
+ ((BinaryNode)nodes[i]).setRightChild(nodes[rightChildIndex]);
+ return toLinkedLogicalNode(nodes, rightChildIndex);
} else if (nodes[i] instanceof TableSubQueryNode) {
- ((TableSubQueryNode)nodes[i]).setSubQuery(nodes[++i]);
+ ((TableSubQueryNode)nodes[i]).setSubQuery(nodes[i+1]);
+ return toLinkedLogicalNode(nodes, i+1);
} else {
- i++;
+ return i+1;
}
+ } else {
+ return -1;
}
- return nodes[0];
}
public LogicalNode getRootNode() {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index fe7d177..b9206a1 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -579,11 +579,11 @@ public class GlobalPlanner {
private LogicalNode handleUnaryNode(GlobalPlanContext context, LogicalNode child, LogicalNode node) {
ExecutionBlock execBlock = context.execBlockMap.remove(child.getPID());
- if (node instanceof UnaryNode) {
- ((UnaryNode)node).setChild(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
- } else if (node instanceof TableSubQueryNode) {
- ((TableSubQueryNode) node).setSubQuery(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
- }
+// if (node instanceof UnaryNode) {
+// ((UnaryNode)node).setChild(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
+// } else if (node instanceof TableSubQueryNode) {
+// ((TableSubQueryNode) node).setSubQuery(execBlock.getPlan().getFirstPlanGroup().toLinkedLogicalNode());
+// }
execBlock.setPlan(node);
context.execBlockMap.put(node.getPID(), execBlock);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
index 49a4b68..bf93f99 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
@@ -24,6 +24,7 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.engine.planner.PlanString;
import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.Target;
+import org.apache.tajo.util.TUtil;
public class TableSubQueryNode extends RelationNode implements Projectable {
@Expose private String tableName;
@@ -97,15 +98,26 @@ public class TableSubQueryNode extends RelationNode implements Projectable {
public boolean equals(Object object) {
if (object instanceof TableSubQueryNode) {
TableSubQueryNode another = (TableSubQueryNode) object;
- return tableName.equals(another.tableName) && subQuery.equals(another.subQuery);
+ return tableName.equals(another.tableName) && TUtil.checkEquals(this.targets, another.targets);
+// && subQuery.equals(another.subQuery);
}
return false;
}
@Override
+ public boolean deepEquals(Object o) {
+ if (o instanceof TableSubQueryNode) {
+ TableSubQueryNode other = (TableSubQueryNode) o;
+ return equals(o) && subQuery.deepEquals(other.subQuery);
+ }
+ return false;
+ }
+
+ @Override
public Object clone() throws CloneNotSupportedException {
TableSubQueryNode newTableSubQueryNode = (TableSubQueryNode) super.clone();
+ newTableSubQueryNode.subQuery = this.subQuery == null ? null : (LogicalNode) this.subQuery.clone();
newTableSubQueryNode.tableName = tableName;
return newTableSubQueryNode;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/global/TestExecutionPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/global/TestExecutionPlan.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/global/TestExecutionPlan.java
index 71d5a9c..981bf9a 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/global/TestExecutionPlan.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/global/TestExecutionPlan.java
@@ -102,4 +102,34 @@ public class TestExecutionPlan {
assertEquals(3, plan.getParentCount(scanNode));
}
+
+ @Test
+ public void testToLinkedLogicalNode() throws CloneNotSupportedException {
+ Schema schema = new Schema();
+ schema.addColumn("id", Type.INT4);
+ schema.addColumn("name", Type.TEXT);
+ schema.addColumn("age", Type.INT2);
+ PIDFactory pidFactory = new PIDFactory();
+
+ LogicalRootNode root1 = new LogicalRootNode(pidFactory.newPID());
+ GroupbyNode groupbyNode = new GroupbyNode(pidFactory.newPID(),
+ new Column[]{schema.getColumn(1), schema.getColumn(2)});
+ ScanNode scanNode = new ScanNode(pidFactory.newPID(),
+ CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
+ ScanNode scanNode2 = new ScanNode(pidFactory.newPID(),
+ CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
+ UnionNode unionNode = new UnionNode(pidFactory.newPID(), groupbyNode, scanNode2);
+ unionNode.setOutSchema(schema);
+ unionNode.setInSchema(schema);
+ TableSubQueryNode tableSubQueryNode = new TableSubQueryNode(pidFactory.newPID(), "test", unionNode);
+ root1.setChild(tableSubQueryNode);
+ groupbyNode.setChild(scanNode);
+
+ LogicalRootNode clone = (LogicalRootNode) root1.clone();
+
+ ExecutionPlan plan = new ExecutionPlan(pidFactory);
+ plan.addPlan(root1);
+
+ assertTrue(clone.getChild().deepEquals(plan.getFirstPlanGroup().toLinkedLogicalNode()));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
index 5885f10..e3e29c2 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
@@ -79,4 +79,32 @@ public class TestCaseByCases {
res.close();
}
}
+
+ @Test
+ public final void testTAJO266Case() throws Exception {
+ ResultSet res = tpch.execute(FileUtil.readTextFile(new File("src/test/queries/tajo266_case.sql")));
+ try {
+ assertTrue(res.next());
+ assertEquals("1", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("1", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("1", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("1", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("2", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("3", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("3", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("3", res.getString(1));
+ assertTrue(res.next());
+ assertEquals("3", res.getString(1));
+ assertFalse(res.next());
+ } finally {
+ res.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/f932f5d0/tajo-core/tajo-core-backend/src/test/queries/tajo266_case.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/queries/tajo266_case.sql b/tajo-core/tajo-core-backend/src/test/queries/tajo266_case.sql
new file mode 100644
index 0000000..5434291
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/queries/tajo266_case.sql
@@ -0,0 +1,9 @@
+SELECT
+ l1.l_orderkey
+
+FROM
+ lineitem AS l1
+
+ INNER JOIN lineitem AS l2
+
+ ON l1.l_orderkey = l2.l_orderkey;
\ No newline at end of file
[06/10] git commit: DAG-execplan
Posted by ji...@apache.org.
DAG-execplan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/24aa0e76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/24aa0e76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/24aa0e76
Branch: refs/heads/DAG-execplan
Commit: 24aa0e76bc1cd941d73a49793228449d6b9931ba
Parents: 77660af
Author: Jihoon Son <ji...@apache.org>
Authored: Mon Dec 30 00:14:09 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Mon Dec 30 00:14:09 2013 +0900
----------------------------------------------------------------------
.../engine/planner/global/ExecutionPlan.java | 120 +++++++++++++------
1 file changed, 85 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/24aa0e76/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
index 80ea564..a6c3bcd 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/ExecutionPlan.java
@@ -50,8 +50,8 @@ public class ExecutionPlan implements GsonObject {
= new SimpleDirectedGraph<Integer, ExecutionPlanEdge>();
@Expose private NavigableMap<Integer, PlanGroup> planGroups = Maps.newTreeMap();
-// private boolean built = false;
private ExecutionPlanBuilder builder = new ExecutionPlanBuilder(this);
+ private final ExecutionPlanComparator comparator = new ExecutionPlanComparator();
public static class PlanGroup {
@Expose private int rootPID;
@@ -148,16 +148,79 @@ public class ExecutionPlan implements GsonObject {
topNode = ((LogicalRootNode)topNode).getChild();
}
+ SameNodeChecker sameNodeChecker = new SameNodeChecker();
+ sameNodeChecker.visit(topNode);
+ LogicalNode sameNode = getSameNode(topNode);
+ if (sameNode != null) {
+ connectChild(topNode, sameNode);
+ topNode = sameNode;
+ }
+
// add group
PlanGroup planGroup = new PlanGroup(topNode.getPID());
planGroup.addNodeAndDescendants(topNode);
planGroups.put(planGroup.rootPID, planGroup);
- topNode = planGroup.nodes.iterator().next();
+ topNode = planGroup.getRootNode();
builder.visit(topNode);
this.add(topNode, terminalNode, EdgeType.SINGLE);
}
+ private LogicalNode getSameNode(LogicalNode node) {
+ for (Entry<Integer, LogicalNode> eachEntry : vertices.entrySet()) {
+ if (node.equals(eachEntry.getValue())) {
+ return eachEntry.getValue();
+ }
+ }
+ return null;
+ }
+
+ private class SameNodeChecker implements LogicalNodeVisitor {
+
+ @Override
+ public void visit(LogicalNode node) {
+ LogicalNode sameNode;
+ if (node instanceof UnaryNode) {
+ UnaryNode unaryNode = (UnaryNode) node;
+ sameNode = getSameNode(unaryNode.getChild());
+ if (sameNode != null) {
+ connectChild(unaryNode.getChild(), sameNode);
+ unaryNode.setChild(sameNode);
+ }
+ } else if (node instanceof BinaryNode) {
+ BinaryNode binaryNode = (BinaryNode) node;
+ sameNode = getSameNode(binaryNode.getLeftChild());
+ if (sameNode != null) {
+ connectChild(binaryNode.getLeftChild(), sameNode);
+ binaryNode.setLeftChild(sameNode);
+ }
+ sameNode = getSameNode(binaryNode.getRightChild());
+ if (sameNode != null) {
+ connectChild(binaryNode.getRightChild(), sameNode);
+ binaryNode.setRightChild(sameNode);
+ }
+ } else if (node instanceof TableSubQueryNode) {
+ TableSubQueryNode tableSubQueryNode = (TableSubQueryNode) node;
+ sameNode = getSameNode(tableSubQueryNode.getSubQuery());
+ if (sameNode != null) {
+ connectChild(tableSubQueryNode.getSubQuery(), sameNode);
+ tableSubQueryNode.setSubQuery(sameNode);
+ }
+ }
+ }
+ }
+
+ private void connectChild(LogicalNode originNode, LogicalNode replaceNode) {
+ if (replaceNode instanceof UnaryNode) {
+ ((UnaryNode) replaceNode).setChild(((UnaryNode)originNode).getChild());
+ } else if (replaceNode instanceof BinaryNode) {
+ ((BinaryNode) replaceNode).setLeftChild(((BinaryNode) originNode).getLeftChild());
+ ((BinaryNode) replaceNode).setRightChild(((BinaryNode) originNode).getRightChild());
+ } else if (replaceNode instanceof TableSubQueryNode) {
+ ((TableSubQueryNode) replaceNode).setSubQuery(((TableSubQueryNode) originNode).getSubQuery());
+ }
+ }
+
public PlanGroup remoteLogicalNodeGroup(int pid) {
// TODO: improve the code
PlanGroup removed = planGroups.remove(pid);
@@ -250,8 +313,7 @@ public class ExecutionPlan implements GsonObject {
return false;
}
- ExecutionPlanComparator comparator = new ExecutionPlanComparator(this, other);
- eq &= comparator.compare();
+ eq &= comparator.compare(this, other);
return eq;
}
return false;
@@ -347,32 +409,10 @@ public class ExecutionPlan implements GsonObject {
}
private static class ExecutionPlanComparator {
- ExecutionPlan plan1;
- ExecutionPlan plan2;
boolean equal = true;
- public ExecutionPlanComparator(ExecutionPlan plan1, ExecutionPlan plan2) {
- this.plan1 = plan1;
- this.plan2 = plan2;
- }
-
- public boolean compare() {
- if (plan1.getChildCount(plan1.terminalNode)
- == plan2.getChildCount(plan2.terminalNode)) {
- Stack<Integer> s1 = new Stack<Integer>();
- Stack<Integer> s2 = new Stack<Integer>();
- int childCount = plan1.getChildCount(plan1.terminalNode);
- for (int i = 0; i < childCount; i++) {
- s1.push(plan1.getTopNode(i).getPID());
- s2.push(plan2.getTopNode(i).getPID());
- }
- return recursiveCompare(s1, s2);
- } else {
- return false;
- }
- }
-
- private boolean recursiveCompare(Stack<Integer> s1, Stack<Integer> s2) {
+ private boolean recursiveCompare(ExecutionPlan plan1, ExecutionPlan plan2,
+ Stack<Integer> s1, Stack<Integer> s2) {
Integer l1 = s1.pop();
Integer l2 = s2.pop();
@@ -386,7 +426,7 @@ public class ExecutionPlan implements GsonObject {
for (Integer child : plan2.graph.getChilds(l2)) {
s2.push(child);
}
- return recursiveCompare(s1, s2);
+ return recursiveCompare(plan1, plan2, s1, s2);
} else {
equal &= true;
}
@@ -398,6 +438,22 @@ public class ExecutionPlan implements GsonObject {
}
return equal;
}
+
+ public boolean compare(ExecutionPlan plan1, ExecutionPlan plan2) {
+ if (plan1.getChildCount(plan1.terminalNode)
+ == plan2.getChildCount(plan2.terminalNode)) {
+ Stack<Integer> s1 = new Stack<Integer>();
+ Stack<Integer> s2 = new Stack<Integer>();
+ int childCount = plan1.getChildCount(plan1.terminalNode);
+ for (int i = 0; i < childCount; i++) {
+ s1.push(plan1.getTopNode(i).getPID());
+ s2.push(plan2.getTopNode(i).getPID());
+ }
+ return recursiveCompare(plan1, plan2, s1, s2);
+ } else {
+ return false;
+ }
+ }
}
private static class ExecutionPlanBuilder implements LogicalNodeVisitor {
@@ -432,12 +488,6 @@ public class ExecutionPlan implements GsonObject {
if (plan.inputContext == null) {
plan.inputContext = new InputContext();
}
- // TODO: update scan and the plan graph
- for (ScanNode eachScan : plan.inputContext.getScanNodes()) {
- if (eachScan.equals(node)) {
- return;
- }
- }
plan.inputContext.addScanNode(node);
}
[04/10] git commit: Revert "TAJO-463: Throw PlanningException when
Creating table with Partition exception COLUMN. (DaeMyung Kang via hyunsik)"
Posted by ji...@apache.org.
Revert "TAJO-463: Throw PlanningException when Creating table with Partition exception COLUMN. (DaeMyung Kang via hyunsik)"
This reverts commit b574cf8c46bbcdf2601fc20ef149d08b0a88742f.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/5c243f1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/5c243f1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/5c243f1a
Branch: refs/heads/DAG-execplan
Commit: 5c243f1a3b01a0e91884d9b894a338c371c5e9a4
Parents: b574cf8
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun Dec 29 22:13:52 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun Dec 29 22:13:52 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 5 +--
.../tajo/engine/planner/LogicalPlanner.java | 29 +++-----------
.../engine/planner/global/GlobalPlanner.java | 17 +++++---
.../org/apache/tajo/client/TestTajoClient.java | 41 +++-----------------
4 files changed, 24 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/5c243f1a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 98d392a..ff45694 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -174,10 +174,7 @@ Release 0.8.0 - unreleased
BUG FIXES
- TAJO-463: Throw PlanningException when Creating table with Partition
- exception COLUMN. (DaeMyung Kang via hyunsik)
-
- TAJO-452: Timstamp literal with fractional seconds results in
+ TAJO-452: Timstamp literal with fractional seconds results in
java.lang.ArrayIndexOutOfBoundsException. (Keuntae Park via hyunsik)
TAJO-454: pass invalid argument to DateTime constructor in LogicalPlanner.
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/5c243f1a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index fea3361..55f4290 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -781,22 +781,12 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return storeNode;
} else {
- Schema tableSchema;
- boolean mergedPartition = false;
- if (expr.hasPartition()) {
- if (expr.getPartition().getPartitionType().equals(PartitionType.COLUMN)) {
- if (((ColumnPartition)expr.getPartition()).isOmitValues()) {
- mergedPartition = true;
- }
- } else {
- throw new PlanningException(String.format("Not supported PartitonType: %s",
- expr.getPartition().getPartitionType()));
- }
- }
- if (mergedPartition) {
+ Schema tableSchema;
+ if (expr.hasPartition() && expr.getPartition().getPartitionType() == PartitionType.COLUMN &&
+ ((ColumnPartition)expr.getPartition()).isOmitValues()) {
ColumnDefinition [] merged = TUtil.concat(expr.getTableElements(),
- ((ColumnPartition)expr.getPartition()).getColumns());
+ ((ColumnPartition)expr.getPartition()).getColumns());
tableSchema = convertTableElementsSchema(merged);
} else {
tableSchema = convertTableElementsSchema(expr.getTableElements());
@@ -818,7 +808,6 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// TODO - it should be configurable.
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
-
if (expr.hasParams()) {
Options options = new Options();
options.putAll(expr.getParams());
@@ -829,15 +818,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setPath(new Path(expr.getLocation()));
}
- if (expr.hasPartition()) {
- if (expr.getPartition().getPartitionType().equals(PartitionType.COLUMN)) {
- createTableNode.setPartitions(convertTableElementsPartition(context, expr));
- } else {
- throw new PlanningException(String.format("Not supported PartitonType: %s",
- expr.getPartition().getPartitionType()));
- }
+ if (expr.hasPartition()) {
+ createTableNode.setPartitions(convertTableElementsPartition(context, expr));
}
-
return createTableNode;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/5c243f1a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 5ab53d8..877a179 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -323,9 +323,7 @@ public class GlobalPlanner {
private ExecutionBlock buildStorePlan(GlobalPlanContext context,
ExecutionBlock childBlock,
- StoreTableNode currentNode)
- throws PlanningException
- {
+ StoreTableNode currentNode) {
PartitionDesc partitionDesc = currentNode.getPartitions();
// if result table is not a partitioned table, directly store it
@@ -353,8 +351,17 @@ public class GlobalPlanner {
channel.setPartitionKey(partitionDesc.getColumns().toArray(columns));
channel.setSchema(childNode.getOutSchema());
channel.setStoreType(storeType);
- } else {
- throw new PlanningException(String.format("Not Supported PartitionsType :%s", partitionsType));
+ } else if (partitionsType == CatalogProtos.PartitionsType.HASH) {
+ channel = new DataChannel(childBlock, currentBlock, HASH_PARTITION,
+ partitionDesc.getNumPartitions());
+ Column[] columns = new Column[partitionDesc.getColumns().size()];
+ channel.setPartitionKey(partitionDesc.getColumns().toArray(columns));
+ channel.setSchema(childNode.getOutSchema());
+ channel.setStoreType(storeType);
+ } else if(partitionsType == CatalogProtos.PartitionsType.RANGE) {
+ // TODO
+ } else if(partitionsType == CatalogProtos.PartitionsType.LIST) {
+ // TODO
}
// 3. create a ScanNode for scanning shuffle data
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/5c243f1a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 6e33d7c..44004d2 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -250,7 +250,7 @@ public class TestTajoClient {
assertTrue(desc.getStats().getNumBytes() > 0);
}
- //@Test
+ @Test
public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -274,7 +274,7 @@ public class TestTajoClient {
assertTrue(hdfs.exists(tablePath));
}
- //@Test
+ @Test
public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -298,7 +298,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- //@Test
+ @Test
public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -322,7 +322,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- //@Test
+ @Test
public final void testCreateAndDropTablePartitionedListByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -347,7 +347,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- //@Test
+ @Test
public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -372,37 +372,6 @@ public class TestTajoClient {
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
-
- @Test
- public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException,
- ServiceException, SQLException {
- TajoConf conf = cluster.getConfiguration();
- final String tableName = "testFailCreateTablePartitionedOtherExceptColumn";
-
- assertFalse(client.existTable(tableName));
-
- String rangeSql = "create table " + tableName + " (deptname text, score int4)";
- rangeSql += "PARTITION BY RANGE (score)";
- rangeSql += "( PARTITION sub_part1 VALUES LESS THAN (2),";
- rangeSql += "PARTITION sub_part2 VALUES LESS THAN (5),";
- rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )";
-
- assertFalse(client.updateQuery(rangeSql));
-
- String listSql = "create table " + tableName + " (deptname text, score int4)";
- listSql += "PARTITION BY LIST (deptname)";
- listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),";
- listSql += "PARTITION sub_part2 VALUES('sales', 'hr') )";
-
- assertFalse(client.updateQuery(listSql));
-
- String hashSql = "create table " + tableName + " (deptname text, score int4)";
- hashSql += "PARTITION BY HASH (deptname)";
- hashSql += "PARTITIONS 2";
-
- assertFalse(client.updateQuery(hashSql));
- }
-
@Test
public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws IOException,
ServiceException, SQLException {
[03/10] git commit: TAJO-463: Throw PlanningException when Creating
table with Partition exception COLUMN. (DaeMyung Kang via hyunsik)
Posted by ji...@apache.org.
TAJO-463: Throw PlanningException when Creating table with Partition exception COLUMN. (DaeMyung Kang via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/b574cf8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/b574cf8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/b574cf8c
Branch: refs/heads/DAG-execplan
Commit: b574cf8c46bbcdf2601fc20ef149d08b0a88742f
Parents: 4c75790
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun Dec 29 22:02:12 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun Dec 29 22:02:12 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 5 ++-
.../tajo/engine/planner/LogicalPlanner.java | 29 +++++++++++---
.../engine/planner/global/GlobalPlanner.java | 17 +++-----
.../org/apache/tajo/client/TestTajoClient.java | 41 +++++++++++++++++---
4 files changed, 68 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b574cf8c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ff45694..98d392a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -174,7 +174,10 @@ Release 0.8.0 - unreleased
BUG FIXES
- TAJO-452: Timstamp literal with fractional seconds results in
+ TAJO-463: Throw PlanningException when Creating table with Partition
+ exception COLUMN. (DaeMyung Kang via hyunsik)
+
+ TAJO-452: Timstamp literal with fractional seconds results in
java.lang.ArrayIndexOutOfBoundsException. (Keuntae Park via hyunsik)
TAJO-454: pass invalid argument to DateTime constructor in LogicalPlanner.
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b574cf8c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 55f4290..fea3361 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -781,12 +781,22 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return storeNode;
} else {
-
Schema tableSchema;
- if (expr.hasPartition() && expr.getPartition().getPartitionType() == PartitionType.COLUMN &&
- ((ColumnPartition)expr.getPartition()).isOmitValues()) {
+ boolean mergedPartition = false;
+ if (expr.hasPartition()) {
+ if (expr.getPartition().getPartitionType().equals(PartitionType.COLUMN)) {
+ if (((ColumnPartition)expr.getPartition()).isOmitValues()) {
+ mergedPartition = true;
+ }
+ } else {
+ throw new PlanningException(String.format("Not supported PartitonType: %s",
+ expr.getPartition().getPartitionType()));
+ }
+ }
+
+ if (mergedPartition) {
ColumnDefinition [] merged = TUtil.concat(expr.getTableElements(),
- ((ColumnPartition)expr.getPartition()).getColumns());
+ ((ColumnPartition)expr.getPartition()).getColumns());
tableSchema = convertTableElementsSchema(merged);
} else {
tableSchema = convertTableElementsSchema(expr.getTableElements());
@@ -808,6 +818,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// TODO - it should be configurable.
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
+
if (expr.hasParams()) {
Options options = new Options();
options.putAll(expr.getParams());
@@ -818,9 +829,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setPath(new Path(expr.getLocation()));
}
- if (expr.hasPartition()) {
- createTableNode.setPartitions(convertTableElementsPartition(context, expr));
+ if (expr.hasPartition()) {
+ if (expr.getPartition().getPartitionType().equals(PartitionType.COLUMN)) {
+ createTableNode.setPartitions(convertTableElementsPartition(context, expr));
+ } else {
+ throw new PlanningException(String.format("Not supported PartitonType: %s",
+ expr.getPartition().getPartitionType()));
+ }
}
+
return createTableNode;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b574cf8c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 877a179..5ab53d8 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -323,7 +323,9 @@ public class GlobalPlanner {
private ExecutionBlock buildStorePlan(GlobalPlanContext context,
ExecutionBlock childBlock,
- StoreTableNode currentNode) {
+ StoreTableNode currentNode)
+ throws PlanningException
+ {
PartitionDesc partitionDesc = currentNode.getPartitions();
// if result table is not a partitioned table, directly store it
@@ -351,17 +353,8 @@ public class GlobalPlanner {
channel.setPartitionKey(partitionDesc.getColumns().toArray(columns));
channel.setSchema(childNode.getOutSchema());
channel.setStoreType(storeType);
- } else if (partitionsType == CatalogProtos.PartitionsType.HASH) {
- channel = new DataChannel(childBlock, currentBlock, HASH_PARTITION,
- partitionDesc.getNumPartitions());
- Column[] columns = new Column[partitionDesc.getColumns().size()];
- channel.setPartitionKey(partitionDesc.getColumns().toArray(columns));
- channel.setSchema(childNode.getOutSchema());
- channel.setStoreType(storeType);
- } else if(partitionsType == CatalogProtos.PartitionsType.RANGE) {
- // TODO
- } else if(partitionsType == CatalogProtos.PartitionsType.LIST) {
- // TODO
+ } else {
+ throw new PlanningException(String.format("Not Supported PartitionsType :%s", partitionsType));
}
// 3. create a ScanNode for scanning shuffle data
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b574cf8c/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 44004d2..6e33d7c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -250,7 +250,7 @@ public class TestTajoClient {
assertTrue(desc.getStats().getNumBytes() > 0);
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -274,7 +274,7 @@ public class TestTajoClient {
assertTrue(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -298,7 +298,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -322,7 +322,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedListByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -347,7 +347,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -372,6 +372,37 @@ public class TestTajoClient {
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
+
+ @Test
+ public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException,
+ ServiceException, SQLException {
+ TajoConf conf = cluster.getConfiguration();
+ final String tableName = "testFailCreateTablePartitionedOtherExceptColumn";
+
+ assertFalse(client.existTable(tableName));
+
+ String rangeSql = "create table " + tableName + " (deptname text, score int4)";
+ rangeSql += "PARTITION BY RANGE (score)";
+ rangeSql += "( PARTITION sub_part1 VALUES LESS THAN (2),";
+ rangeSql += "PARTITION sub_part2 VALUES LESS THAN (5),";
+ rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )";
+
+ assertFalse(client.updateQuery(rangeSql));
+
+ String listSql = "create table " + tableName + " (deptname text, score int4)";
+ listSql += "PARTITION BY LIST (deptname)";
+ listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),";
+ listSql += "PARTITION sub_part2 VALUES('sales', 'hr') )";
+
+ assertFalse(client.updateQuery(listSql));
+
+ String hashSql = "create table " + tableName + " (deptname text, score int4)";
+ hashSql += "PARTITION BY HASH (deptname)";
+ hashSql += "PARTITIONS 2";
+
+ assertFalse(client.updateQuery(hashSql));
+ }
+
@Test
public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws IOException,
ServiceException, SQLException {
[07/10] git commit: TAJO-460: CTAS statement should support
partitioned table. (Min Zhou via hyunsik)
Posted by ji...@apache.org.
TAJO-460: CTAS statement should support partitioned table. (Min Zhou via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/d39bb998
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/d39bb998
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/d39bb998
Branch: refs/heads/DAG-execplan
Commit: d39bb99809384c25214f4847c8de06e9ac012e98
Parents: 2a41930
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Dec 30 12:41:09 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Dec 30 12:41:09 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../tajo/catalog/partition/PartitionDesc.java | 4 +
.../tajo/engine/planner/LogicalPlanner.java | 10 ++
.../apache/tajo/engine/query/QueryContext.java | 14 +++
.../org/apache/tajo/master/GlobalEngine.java | 3 +
.../apache/tajo/master/querymaster/Query.java | 5 +
.../apache/tajo/engine/query/TestCTASQuery.java | 97 ++++++++++++++++++++
.../create_partitioned_table_as_select.sql | 6 ++
8 files changed, 142 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ac27e63..fae9d1d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,9 @@ Release 0.8.0 - unreleased
NEW FEATURES
+ TAJO-460: CTAS statement should support partitioned table.
+ (Min Zhou via hyunsik)
+
TAJO-381: Implement find_in_set function. (Jae Young Lee via hyunsik)
TAJO-439: Time literal support. (DaeMyung Kang via jihoon)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
index fbec807..792d642 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
@@ -191,4 +191,8 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro
return CatalogGsonHelper.toJson(this, PartitionDesc.class);
}
+
+ public static PartitionDesc fromJson(String strVal) {
+ return strVal != null ? CatalogGsonHelper.fromJson(strVal, PartitionDesc.class) : null;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index fea3361..04bc27b 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -759,9 +759,11 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
storeNode.setChild(subQuery);
if (expr.hasTableElements()) {
+ // CREATE TABLE tbl(col1 type, col2 type) AS SELECT ...
Schema schema = convertTableElementsSchema(expr.getTableElements());
storeNode.setOutSchema(schema);
} else {
+ // CREATE TABLE tbl AS SELECT ...
storeNode.setOutSchema(subQuery.getOutSchema());
}
storeNode.setInSchema(subQuery.getOutSchema());
@@ -779,6 +781,10 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
storeNode.setOptions(options);
}
+ if (expr.hasPartition()) {
+ storeNode.setPartitions(convertTableElementsPartition(context, expr));
+ }
+
return storeNode;
} else {
Schema tableSchema;
@@ -1017,12 +1023,16 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Schema targetSchema = new Schema();
if (expr.hasTargetColumns()) {
+ // INSERT OVERWRITE INTO TABLE tbl(col1 type, col2 type) SELECT ...
String [] targetColumnNames = expr.getTargetColumns();
for (int i = 0; i < targetColumnNames.length; i++) {
Column targetColumn = context.plan.resolveColumn(context.block, null, new ColumnReferenceExpr(targetColumnNames[i]));
targetSchema.addColumn(targetColumn);
}
} else {
+ // use the output schema of select clause as target schema
+ // if didn't specific target columns like the way below,
+ // INSERT OVERWRITE INTO TABLE tbl SELECT ...
Schema targetTableSchema = desc.getSchema();
for (int i = 0; i < subQuery.getOutSchema().getColumnNum(); i++) {
targetSchema.addColumn(targetTableSchema.getColumn(i));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/QueryContext.java
index 26fd8c0..e710f9d 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/QueryContext.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/QueryContext.java
@@ -20,6 +20,7 @@ package org.apache.tajo.engine.query;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.Options;
+import org.apache.tajo.catalog.partition.PartitionDesc;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.logical.NodeType;
@@ -36,6 +37,7 @@ public class QueryContext extends Options {
public static final String OUTPUT_TABLE_NAME = "tajo.query.output.table";
public static final String OUTPUT_TABLE_PATH = "tajo.query.output.path";
+ public static final String OUTPUT_PARTITIONS = "tajo.query.output.partitions";
public static final String OUTPUT_OVERWRITE = "tajo.query.output.overwrite";
public static final String OUTPUT_AS_DIRECTORY = "tajo.query.output.asdirectory";
@@ -128,6 +130,18 @@ public class QueryContext extends Options {
return strVal != null ? new Path(strVal) : null;
}
+ public boolean hasPartitions() {
+ return get(OUTPUT_PARTITIONS) != null;
+ }
+
+ public void setPartitions(PartitionDesc partitionDesc) {
+ put(OUTPUT_PARTITIONS, partitionDesc != null ? partitionDesc.toJson() : null);
+ }
+
+ public PartitionDesc getPartitions() {
+ return PartitionDesc.fromJson(get(OUTPUT_PARTITIONS));
+ }
+
public void setOutputOverwrite() {
setBool(OUTPUT_OVERWRITE, true);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
index d17ec91..7336f2f 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -371,6 +371,9 @@ public class GlobalEngine extends AbstractService {
String tableName = storeTableNode.getTableName();
queryContext.setOutputTable(tableName);
queryContext.setOutputPath(new Path(TajoConf.getWarehouseDir(context.getConf()), tableName));
+ if(storeTableNode.getPartitions() != null) {
+ queryContext.setPartitions(storeTableNode.getPartitions());
+ }
queryContext.setCreateTable();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 79ae34d..f8c335b 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -410,6 +410,11 @@ public class Query implements EventHandler<QueryEvent> {
finalTableDesc = updatingTable;
}
}
+
+ if(queryContext.hasPartitions()) {
+ finalTableDesc.setPartitions(queryContext.getPartitions());
+ }
+
return finalTableDesc;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
new file mode 100644
index 0000000..1ff3b97
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.IntegrationTest;
+import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.TpchTestBase;
+import org.apache.tajo.catalog.CatalogService;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.partition.PartitionDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test CREATE TABLE AS SELECT statements
+ */
+@Category(IntegrationTest.class)
+public class TestCTASQuery {
+ private static TpchTestBase tpch;
+ public TestCTASQuery() throws IOException {
+ super();
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ tpch = TpchTestBase.getInstance();
+ }
+
+ @Test
+ public final void testCtasWithColumnedPartition() throws Exception {
+ String tableName ="testCtasWithColumnedPartition";
+ tpch.execute(
+ "create table " + tableName
+ + " (col1 int4, col2 int4) partition by column(key float8) "
+ + " as select l_orderkey as col1, l_partkey as col2, l_quantity as key from lineitem");
+
+ TajoTestingCluster cluster = tpch.getTestingCluster();
+ CatalogService catalog = cluster.getMaster().getCatalog();
+ TableDesc desc = catalog.getTableDesc(tableName);
+ assertTrue(catalog.existsTable(tableName));
+ PartitionDesc partitionDesc = desc.getPartitions();
+ assertEquals(partitionDesc.getPartitionsType(), CatalogProtos.PartitionsType.COLUMN);
+ assertEquals("key", partitionDesc.getColumns().get(0).getColumnName());
+
+ FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration());
+ Path path = desc.getPath();
+ assertTrue(fs.isDirectory(path));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0")));
+ assertEquals(5, desc.getStats().getNumRows().intValue());
+
+ ResultSet res = tpch.execute(
+ "select distinct * from " + tableName + " where (key = 45.0 or key = 38.0)");
+
+ Map<Double, int []> resultRows1 = Maps.newHashMap();
+ resultRows1.put(45.0d, new int[]{3, 2});
+ resultRows1.put(38.0d, new int[]{2, 2});
+
+ for (int i = 0; i < 3 && res.next(); i++) {
+ assertEquals(resultRows1.get(res.getDouble(3))[0], res.getInt(1));
+ assertEquals(resultRows1.get(res.getDouble(3))[1], res.getInt(2));
+ }
+ res.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d39bb998/tajo-core/tajo-core-backend/src/test/queries/create_partitioned_table_as_select.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/queries/create_partitioned_table_as_select.sql b/tajo-core/tajo-core-backend/src/test/queries/create_partitioned_table_as_select.sql
new file mode 100644
index 0000000..09b14eb
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/queries/create_partitioned_table_as_select.sql
@@ -0,0 +1,6 @@
+CREATE TABLE sales ( col1 int, col2 int)
+PARTITION BY COLUMN (col3 int, col4 float, col5 text)
+AS SELECT col1, col2, col3, col4, col5 FROM sales_src
+ WHERE col1 > 16
+
+
[05/10] git commit: TAJO-455: Throw PlanningException when Creating
table with Partition exception COLUMN. (DaeMyung Kang via hyunsik)
Posted by ji...@apache.org.
TAJO-455: Throw PlanningException when Creating table with Partition exception COLUMN. (DaeMyung Kang via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/2a419306
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/2a419306
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/2a419306
Branch: refs/heads/DAG-execplan
Commit: 2a419306a5b90e74b4f9653ce8a439c9e373f389
Parents: 5c243f1
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun Dec 29 22:16:12 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun Dec 29 22:16:12 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../tajo/engine/planner/LogicalPlanner.java | 29 +++++++++++---
.../engine/planner/global/GlobalPlanner.java | 17 +++-----
.../org/apache/tajo/client/TestTajoClient.java | 41 +++++++++++++++++---
4 files changed, 67 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a419306/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ff45694..ac27e63 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -174,6 +174,9 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-455: Throw PlanningException when Creating table with Partition
+ exception COLUMN. (DaeMyung Kang via hyunsik)
+
TAJO-452: Timstamp literal with fractional seconds results in
java.lang.ArrayIndexOutOfBoundsException. (Keuntae Park via hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a419306/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 55f4290..fea3361 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -781,12 +781,22 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return storeNode;
} else {
-
Schema tableSchema;
- if (expr.hasPartition() && expr.getPartition().getPartitionType() == PartitionType.COLUMN &&
- ((ColumnPartition)expr.getPartition()).isOmitValues()) {
+ boolean mergedPartition = false;
+ if (expr.hasPartition()) {
+ if (expr.getPartition().getPartitionType().equals(PartitionType.COLUMN)) {
+ if (((ColumnPartition)expr.getPartition()).isOmitValues()) {
+ mergedPartition = true;
+ }
+ } else {
+ throw new PlanningException(String.format("Not supported PartitonType: %s",
+ expr.getPartition().getPartitionType()));
+ }
+ }
+
+ if (mergedPartition) {
ColumnDefinition [] merged = TUtil.concat(expr.getTableElements(),
- ((ColumnPartition)expr.getPartition()).getColumns());
+ ((ColumnPartition)expr.getPartition()).getColumns());
tableSchema = convertTableElementsSchema(merged);
} else {
tableSchema = convertTableElementsSchema(expr.getTableElements());
@@ -808,6 +818,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// TODO - it should be configurable.
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
+
if (expr.hasParams()) {
Options options = new Options();
options.putAll(expr.getParams());
@@ -818,9 +829,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setPath(new Path(expr.getLocation()));
}
- if (expr.hasPartition()) {
- createTableNode.setPartitions(convertTableElementsPartition(context, expr));
+ if (expr.hasPartition()) {
+ if (expr.getPartition().getPartitionType().equals(PartitionType.COLUMN)) {
+ createTableNode.setPartitions(convertTableElementsPartition(context, expr));
+ } else {
+ throw new PlanningException(String.format("Not supported PartitonType: %s",
+ expr.getPartition().getPartitionType()));
+ }
}
+
return createTableNode;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a419306/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 877a179..5ab53d8 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -323,7 +323,9 @@ public class GlobalPlanner {
private ExecutionBlock buildStorePlan(GlobalPlanContext context,
ExecutionBlock childBlock,
- StoreTableNode currentNode) {
+ StoreTableNode currentNode)
+ throws PlanningException
+ {
PartitionDesc partitionDesc = currentNode.getPartitions();
// if result table is not a partitioned table, directly store it
@@ -351,17 +353,8 @@ public class GlobalPlanner {
channel.setPartitionKey(partitionDesc.getColumns().toArray(columns));
channel.setSchema(childNode.getOutSchema());
channel.setStoreType(storeType);
- } else if (partitionsType == CatalogProtos.PartitionsType.HASH) {
- channel = new DataChannel(childBlock, currentBlock, HASH_PARTITION,
- partitionDesc.getNumPartitions());
- Column[] columns = new Column[partitionDesc.getColumns().size()];
- channel.setPartitionKey(partitionDesc.getColumns().toArray(columns));
- channel.setSchema(childNode.getOutSchema());
- channel.setStoreType(storeType);
- } else if(partitionsType == CatalogProtos.PartitionsType.RANGE) {
- // TODO
- } else if(partitionsType == CatalogProtos.PartitionsType.LIST) {
- // TODO
+ } else {
+ throw new PlanningException(String.format("Not Supported PartitionsType :%s", partitionsType));
}
// 3. create a ScanNode for scanning shuffle data
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a419306/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 44004d2..6e33d7c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -250,7 +250,7 @@ public class TestTajoClient {
assertTrue(desc.getStats().getNumBytes() > 0);
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -274,7 +274,7 @@ public class TestTajoClient {
assertTrue(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -298,7 +298,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -322,7 +322,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedListByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -347,7 +347,7 @@ public class TestTajoClient {
assertFalse(hdfs.exists(tablePath));
}
- @Test
+ //@Test
public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws IOException,
ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
@@ -372,6 +372,37 @@ public class TestTajoClient {
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
+
+ @Test
+ public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException,
+ ServiceException, SQLException {
+ TajoConf conf = cluster.getConfiguration();
+ final String tableName = "testFailCreateTablePartitionedOtherExceptColumn";
+
+ assertFalse(client.existTable(tableName));
+
+ String rangeSql = "create table " + tableName + " (deptname text, score int4)";
+ rangeSql += "PARTITION BY RANGE (score)";
+ rangeSql += "( PARTITION sub_part1 VALUES LESS THAN (2),";
+ rangeSql += "PARTITION sub_part2 VALUES LESS THAN (5),";
+ rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )";
+
+ assertFalse(client.updateQuery(rangeSql));
+
+ String listSql = "create table " + tableName + " (deptname text, score int4)";
+ listSql += "PARTITION BY LIST (deptname)";
+ listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),";
+ listSql += "PARTITION sub_part2 VALUES('sales', 'hr') )";
+
+ assertFalse(client.updateQuery(listSql));
+
+ String hashSql = "create table " + tableName + " (deptname text, score int4)";
+ hashSql += "PARTITION BY HASH (deptname)";
+ hashSql += "PARTITIONS 2";
+
+ assertFalse(client.updateQuery(hashSql));
+ }
+
@Test
public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws IOException,
ServiceException, SQLException {
[08/10] git commit: Merge branch 'master' of
http://git-wip-us.apache.org/repos/asf/incubator-tajo into DAG-execplan
Posted by ji...@apache.org.
Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/incubator-tajo into DAG-execplan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/9190c2d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/9190c2d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/9190c2d9
Branch: refs/heads/DAG-execplan
Commit: 9190c2d9029910ba4394cc00577e1acbfdd18b5b
Parents: 24aa0e7 d39bb99
Author: Jihoon Son <ji...@apache.org>
Authored: Mon Dec 30 16:47:12 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Mon Dec 30 16:47:12 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 8 ++
.../tajo/catalog/partition/PartitionDesc.java | 4 +
.../tajo/engine/function/string/FindInSet.java | 104 +++++++++++++++++++
.../tajo/engine/planner/LogicalPlanner.java | 39 +++++--
.../engine/planner/global/GlobalPlanner.java | 17 +--
.../apache/tajo/engine/query/QueryContext.java | 14 +++
.../org/apache/tajo/master/GlobalEngine.java | 3 +
.../java/org/apache/tajo/master/TajoMaster.java | 6 ++
.../apache/tajo/master/querymaster/Query.java | 5 +
.../org/apache/tajo/client/TestTajoClient.java | 41 +++++++-
.../apache/tajo/engine/eval/ExprTestBase.java | 14 ++-
.../TestStringOperatorsAndFunctions.java | 29 +++++-
.../apache/tajo/engine/query/TestCTASQuery.java | 97 +++++++++++++++++
.../create_partitioned_table_as_select.sql | 6 ++
14 files changed, 359 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/9190c2d9/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------