You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/01/04 14:32:21 UTC
git commit: TAJO-469: CTAS with no column definition will get a NPE.
(Min Zhou via hyunsik)
Updated Branches:
refs/heads/master bb7e6b6b3 -> 7aba3958e
TAJO-469: CTAS with no column definition will get a NPE. (Min Zhou via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/7aba3958
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/7aba3958
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/7aba3958
Branch: refs/heads/master
Commit: 7aba3958e2d7f3ae5f82cc50928f778566430b1c
Parents: bb7e6b6
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sat Jan 4 22:32:00 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sat Jan 4 22:32:00 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../org/apache/tajo/engine/parser/SQLParser.g4 | 4 +-
.../tajo/engine/planner/LogicalPlanner.java | 23 +++++++---
.../engine/planner/global/GlobalPlanner.java | 3 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 46 +++++++++++++++++++-
5 files changed, 67 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7aba3958/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f3e74ea..48bfa3d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -192,6 +192,9 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-469: CTAS with no column definition will get a NPE.
+ (Min Zhou via hyunsik)
+
TAJO-467: Too many open FD when master failed. (hyoungjunkim via hyunsik)
TAJO-455: Throw PlanningException when Creating table with Partition
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7aba3958/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 06a2b7a..fbae175 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -66,8 +66,10 @@ index_statement
create_table_statement
: CREATE EXTERNAL TABLE table_name table_elements USING file_type=Identifier
(param_clause)? (table_partitioning_clauses)? (LOCATION path=Character_String_Literal)
- | CREATE TABLE table_name (table_elements)? (USING file_type=Identifier)?
+ | CREATE TABLE table_name table_elements (USING file_type=Identifier)?
(param_clause)? (table_partitioning_clauses)? (AS query_expression)?
+ | CREATE TABLE table_name (USING file_type=Identifier)?
+ (param_clause)? (table_partitioning_clauses)? AS query_expression
;
table_elements
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7aba3958/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 04bc27b..7349c0d 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -758,15 +758,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
storeNode.setCreateTable();
storeNode.setChild(subQuery);
- if (expr.hasTableElements()) {
- // CREATE TABLE tbl(col1 type, col2 type) AS SELECT ...
- Schema schema = convertTableElementsSchema(expr.getTableElements());
- storeNode.setOutSchema(schema);
- } else {
+ storeNode.setInSchema(subQuery.getOutSchema());
+ if(!expr.hasTableElements()) {
// CREATE TABLE tbl AS SELECT ...
- storeNode.setOutSchema(subQuery.getOutSchema());
+ expr.setTableElements(convertSchemaToTableElements(subQuery.getOutSchema()));
}
- storeNode.setInSchema(subQuery.getOutSchema());
+ // else CREATE TABLE tbl(col1 type, col2 type) AS SELECT ...
+ storeNode.setOutSchema(convertTableElementsSchema(expr.getTableElements()));
if (expr.hasStorageType()) {
storeNode.setStorageType(CatalogUtil.getStoreType(expr.getStorageType()));
@@ -977,6 +975,17 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return schema;
}
+ private ColumnDefinition[] convertSchemaToTableElements(Schema schema) {
+ List<Column> columns = schema.getColumns();
+ ColumnDefinition[] columnDefinitions = new ColumnDefinition[columns.size()];
+ for(int i = 0; i < columns.size(); i ++) {
+ Column col = columns.get(i);
+ columnDefinitions[i] = new ColumnDefinition(col.getColumnName(), col.getDataType().getType().name());
+ }
+
+ return columnDefinitions;
+ }
+
private Collection<Column> convertTableElementsColumns(CreateTable.ColumnDefinition [] elements,
ColumnReferenceExpr[] references) {
List<Column> columnList = TUtil.newList();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7aba3958/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index abe6af3..2b3ecea 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -340,8 +340,7 @@ public class GlobalPlanner {
private ExecutionBlock buildStorePlan(GlobalPlanContext context,
ExecutionBlock childBlock,
StoreTableNode currentNode)
- throws PlanningException
- {
+ throws PlanningException {
PartitionDesc partitionDesc = currentNode.getPartitions();
// if result table is not a partitioned table, directly store it
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7aba3958/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index 1ff3b97..f879c2c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -39,6 +39,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
/**
* Test CREATE TABLE AS SELECT statements
*/
@@ -55,12 +56,53 @@ public class TestCTASQuery {
}
@Test
+ public final void testCtasWithoutTableDefinition() throws Exception {
+ String tableName ="testCtasWithoutTableDefinition";
+ tpch.execute(
+ "create table " + tableName
+ + " partition by column(key float8) "
+ + " as select l_orderkey as col1, l_partkey as col2, l_quantity as key from lineitem");
+
+ TajoTestingCluster cluster = tpch.getTestingCluster();
+ CatalogService catalog = cluster.getMaster().getCatalog();
+ TableDesc desc = catalog.getTableDesc(tableName);
+ assertTrue(catalog.existsTable(tableName));
+ assertTrue(desc.getSchema().contains("testCtasWithoutTableDefinition.col1"));
+ PartitionDesc partitionDesc = desc.getPartitions();
+ assertEquals(partitionDesc.getPartitionsType(), CatalogProtos.PartitionsType.COLUMN);
+ assertEquals("key", partitionDesc.getColumns().get(0).getColumnName());
+
+ FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration());
+ Path path = desc.getPath();
+ assertTrue(fs.isDirectory(path));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0")));
+ assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0")));
+ assertEquals(5, desc.getStats().getNumRows().intValue());
+
+ ResultSet res = tpch.execute(
+ "select distinct * from " + tableName + " where (key = 45.0 or key = 38.0)");
+
+ Map<Double, int []> resultRows1 = Maps.newHashMap();
+ resultRows1.put(45.0d, new int[]{3, 2});
+ resultRows1.put(38.0d, new int[]{2, 2});
+
+ for (int i = 0; i < 3 && res.next(); i++) {
+ assertEquals(resultRows1.get(res.getDouble(3))[0], res.getInt(1));
+ assertEquals(resultRows1.get(res.getDouble(3))[1], res.getInt(2));
+ }
+ res.close();
+ }
+
+ @Test
public final void testCtasWithColumnedPartition() throws Exception {
String tableName ="testCtasWithColumnedPartition";
tpch.execute(
"create table " + tableName
- + " (col1 int4, col2 int4) partition by column(key float8) "
- + " as select l_orderkey as col1, l_partkey as col2, l_quantity as key from lineitem");
+ + " (col1 int4, col2 int4) partition by column(key float8) "
+ + " as select l_orderkey as col1, l_partkey as col2, l_quantity as key from lineitem");
TajoTestingCluster cluster = tpch.getTestingCluster();
CatalogService catalog = cluster.getMaster().getCatalog();