You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2014/03/25 15:45:37 UTC
git commit: TAJO-705: CTAS always stores tables with CSV storage type
into catalog. (jinho)
Repository: tajo
Updated Branches:
refs/heads/master 4209b83c2 -> 6a167aeaa
TAJO-705: CTAS always stores tables with CSV storage type into catalog. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6a167aea
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6a167aea
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6a167aea
Branch: refs/heads/master
Commit: 6a167aeaaea4ed1735e808875037b9796713bc1f
Parents: 4209b83
Author: jinossy <ji...@gmail.com>
Authored: Tue Mar 25 23:42:26 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Tue Mar 25 23:42:26 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../org/apache/tajo/catalog/CatalogUtil.java | 14 +++++--
.../tajo/engine/planner/LogicalPlanner.java | 6 ++-
.../apache/tajo/master/querymaster/Query.java | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 41 +++++++++++++++++---
.../queries/TestCTASQuery/CtasWithOptions.sql | 13 +++++++
.../queries/TestCTASQuery/CtasWithStoreType.sql | 12 ++++++
.../TestCTASQuery/testCtasWithOptions.sql | 1 +
.../TestCTASQuery/testCtasWithStoreType.sql | 1 +
.../TestCTASQuery/testCtasWithOptions.result | 5 +++
.../TestCTASQuery/testCtasWithStoreType.result | 5 +++
11 files changed, 92 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8c0254d..4609d08 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -283,6 +283,9 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-705: CTAS always stores tables with CSV storage type into catalog.
+ (jinho)
+
TAJO-693: StatusUpdateTransition in QueryUnitAttempt handles TA_UPDATE
incorrectly. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 2ddbb17..5ad2eca 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -183,11 +183,9 @@ public class CatalogUtil {
return StoreType.CSV;
} else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) {
return StoreType.RAW;
- } else if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) {
- return StoreType.CSV;
} else if (typeStr.equalsIgnoreCase(StoreType.ROWFILE.name())) {
return StoreType.ROWFILE;
- }else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) {
+ } else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) {
return StoreType.RCFILE;
} else if (typeStr.equalsIgnoreCase(StoreType.TREVNI.name())) {
return StoreType.TREVNI;
@@ -195,6 +193,16 @@ public class CatalogUtil {
return null;
}
}
+ public static Options newOptionsWithDefault(StoreType type) {
+ Options options = new Options();
+ if(StoreType.CSV == type){
+ options.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.CSVFILE_DELIMITER_DEFAULT);
+ } else if(StoreType.RCFILE == type){
+ options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.RCFILE_BINARY_SERDE);
+ }
+
+ return options;
+ }
public static TableMeta newTableMeta(StoreType type) {
return new TableMeta(type, new Options());
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 48e3c93..c479f1c 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -1313,12 +1313,14 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
+ // Set default options to be created.
+ Options options = CatalogUtil.newOptionsWithDefault(createTableNode.getStorageType());
if (expr.hasParams()) {
- Options options = new Options();
options.putAll(expr.getParams());
- createTableNode.setOptions(options);
}
+ createTableNode.setOptions(options);
+
if (expr.hasPartition()) {
if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
createTableNode.setPartitionMethod(getPartitionMethod(context, expr.getTableName(), expr.getPartitionMethod()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 8467b4b..3a4df7b 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -482,10 +482,10 @@ public class Query implements EventHandler<QueryEvent> {
Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception {
CatalogService catalog = context.getWorkerContext().getCatalog();
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
- TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getResultStats();
CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan();
+ TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions());
TableDesc tableDescTobeCreated =
new TableDesc(
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index e61f509..82c4be9 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -19,15 +19,14 @@
package org.apache.tajo.engine.query;
import com.google.common.collect.Maps;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.IntegrationTest;
import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.catalog.CatalogService;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.junit.Test;
@@ -38,8 +37,7 @@ import java.util.Map;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.catalog.CatalogUtil.buildFQName;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
/**
@@ -178,4 +176,37 @@ public class TestCTASQuery extends QueryTestCaseBase {
resultSetToString(res2);
res2.close();
}
+
+ @Test
+ public final void testCtasWithStoreType() throws Exception {
+ ResultSet res = executeFile("CtasWithStoreType.sql");
+ res.close();
+
+ ResultSet res2 = executeQuery();
+ resultSetToString(res2);
+ res2.close();
+
+ TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
+ assertNotNull(desc);
+ assertEquals(CatalogProtos.StoreType.RCFILE, desc.getMeta().getStoreType());
+ }
+
+ @Test
+ public final void testCtasWithOptions() throws Exception {
+ ResultSet res = executeFile("CtasWithOptions.sql");
+ res.close();
+
+ ResultSet res2 = executeQuery();
+ resultSetToString(res2);
+ res2.close();
+
+ TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
+ assertNotNull(desc);
+ assertEquals(CatalogProtos.StoreType.CSV, desc.getMeta().getStoreType());
+
+
+ Options options = desc.getMeta().getOptions();
+ assertNotNull(options);
+ assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(CatalogConstants.CSVFILE_DELIMITER));
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
new file mode 100644
index 0000000..a393b9d
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
@@ -0,0 +1,13 @@
+create table testCtasWithOptions (col1 float, col2 float)
+using csv with ('csvfile.delimiter'='\u0001') as
+select
+ sum(l_orderkey) as total1,
+ avg(l_partkey) as total2
+from
+ lineitem
+group by
+ l_quantity
+order by
+ l_quantity
+limit
+ 3;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
new file mode 100644
index 0000000..bee1a89
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
@@ -0,0 +1,12 @@
+create table testCtasWithStoreType (col1 float, col2 float) using rcfile as
+select
+ sum(l_orderkey) as total1,
+ avg(l_partkey) as total2
+from
+ lineitem
+group by
+ l_quantity
+order by
+ l_quantity
+limit
+ 3;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
new file mode 100644
index 0000000..ac25e10
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
@@ -0,0 +1 @@
+select * from testCtasWithOptions;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
new file mode 100644
index 0000000..9e7e337
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
@@ -0,0 +1 @@
+select * from testCtasWithStoreType;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
new file mode 100644
index 0000000..ea78854
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
@@ -0,0 +1,5 @@
+col1,col2,key
+-------------------------------
+1.0,1.0,17.0
+1.0,1.0,36.0
+2.0,2.0,38.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
new file mode 100644
index 0000000..ea78854
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
@@ -0,0 +1,5 @@
+col1,col2,key
+-------------------------------
+1.0,1.0,17.0
+1.0,1.0,36.0
+2.0,2.0,38.0
\ No newline at end of file