You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/05/27 20:33:08 UTC
git commit: TAJO-699: Create a table using LIKE. (Prafulla T via
hyunsik)
Repository: tajo
Updated Branches:
refs/heads/master 31e247cd6 -> a191b1dc7
TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik)
Closes #21
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a191b1dc
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a191b1dc
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a191b1dc
Branch: refs/heads/master
Commit: a191b1dc7ad3189c825069084fc6fc6167a4eb87
Parents: 31e247c
Author: Hyunsik Choi <hy...@apache.org>
Authored: Wed May 28 03:01:17 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Wed May 28 03:01:17 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/algebra/CreateTable.java | 11 ++
.../org/apache/tajo/engine/parser/SQLParser.g4 | 3 +-
.../tajo/engine/parser/HiveQLAnalyzer.java | 7 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 6 +-
.../tajo/engine/planner/LogicalPlanner.java | 31 +++-
.../tajo/engine/parser/TestHiveQLAnalyzer.java | 13 ++
.../tajo/engine/parser/TestSQLAnalyzer.java | 9 +
.../tajo/engine/query/TestCreateTable.java | 176 +++++++++++++++++++
.../queries/default/create_table_like_1.sql | 1 +
10 files changed, 255 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8107f81..66b5641 100644
--- a/CHANGES
+++ b/CHANGES
@@ -15,6 +15,8 @@ Release 0.9.0 - unreleased
IMPROVEMENT
+ TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik)
+
TAJO-825: Datetime type refactoring. (Hyoungjun Kim via jihoon)
TAJO-811: add simple fifo scheduler support. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
index f60b571..c74677d 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
@@ -47,6 +47,8 @@ public class CreateTable extends Expr {
private PartitionMethodDescExpr partition;
@Expose @SerializedName("IfNotExists")
private boolean ifNotExists;
+ @Expose @SerializedName("LikeParentTable")
+ private String likeParentTable;
public CreateTable(final String tableName, boolean ifNotExists) {
super(OpType.CreateTable);
@@ -147,6 +149,15 @@ public class CreateTable extends Expr {
return ifNotExists;
}
+ public void setLikeParentTable(String parentTable) {
+ this.likeParentTable = parentTable;
+ }
+
+ public String getLikeParentTableName() {
+ return likeParentTable;
+ }
+
+
@Override
public int hashCode() {
return Objects.hashCode(
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 0076794..9570457 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -94,6 +94,7 @@ create_table_statement
(param_clause)? (table_partitioning_clauses)? (AS query_expression)?
| CREATE TABLE (if_not_exists)? table_name (USING file_type=identifier)?
(param_clause)? (table_partitioning_clauses)? AS query_expression
+ | CREATE TABLE (if_not_exists)? table_name LIKE like_table_name=table_name
;
table_elements
@@ -804,7 +805,7 @@ boolean_primary
;
boolean_predicand
- : parenthesized_boolean_value_expression
+ : parenthesized_boolean_value_expression
| nonparenthesized_value_expression_primary
;
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
index de4b159..2cec22b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
@@ -1425,6 +1425,11 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> {
createTable.setExternal();
}
+ if(ctx.KW_LIKE() != null) {
+ createTable.setLikeParentTable(ctx.likeName.getText());
+ return createTable;
+ }
+
if (ctx.tableFileFormat() != null) {
if (ctx.tableFileFormat().KW_RCFILE() != null) {
createTable.setStorageType("rcfile");
@@ -1548,4 +1553,4 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> {
return Character.toUpperCase((char) returnChar);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index e7de8f6..fe65d47 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1006,8 +1006,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
@Override
public Expr visitCreate_table_statement(SQLParser.Create_table_statementContext ctx) {
- String tableName = ctx.table_name().getText();
+ String tableName = ctx.table_name(0).getText();
CreateTable createTable = new CreateTable(tableName, checkIfExist(ctx.if_not_exists()));
+ if(checkIfExist(ctx.LIKE())) {
+ createTable.setLikeParentTable(ctx.like_table_name.getText());
+ return createTable;
+ }
if (checkIfExist(ctx.EXTERNAL())) {
createTable.setExternal();
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 0780d4f..76dae24 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -1315,6 +1315,33 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return dropDatabaseNode;
}
+ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, CreateTableNode createTableNode)
+ throws PlanningException {
+ String parentTableName = expr.getLikeParentTableName();
+
+ if (CatalogUtil.isFQTableName(parentTableName) == false) {
+ parentTableName =
+ CatalogUtil.buildFQName(context.session.getCurrentDatabase(),
+ parentTableName);
+ }
+ TableDesc parentTableDesc = catalog.getTableDesc(parentTableName);
+ if(parentTableDesc == null)
+ throw new PlanningException("Table '"+parentTableName+"' does not exist");
+ PartitionMethodDesc partitionDesc = parentTableDesc.getPartitionMethod();
+ createTableNode.setTableSchema(parentTableDesc.getSchema());
+ createTableNode.setPartitionMethod(partitionDesc);
+
+ createTableNode.setStorageType(parentTableDesc.getMeta().getStoreType());
+ createTableNode.setOptions(parentTableDesc.getMeta().getOptions());
+
+ createTableNode.setExternal(parentTableDesc.isExternal());
+ if(parentTableDesc.isExternal()) {
+ createTableNode.setPath(parentTableDesc.getPath());
+ }
+ return createTableNode;
+ }
+
+
@Override
public LogicalNode visitCreateTable(PlanContext context, Stack<Expr> stack, CreateTable expr)
throws PlanningException {
@@ -1329,7 +1356,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setTableName(
CatalogUtil.buildFQName(context.session.getCurrentDatabase(), expr.getTableName()));
}
-
+ // This is CREATE TABLE <tablename> LIKE <parentTable>
+ if(expr.getLikeParentTableName() != null)
+ return handleCreateTableLike(context, expr, createTableNode);
if (expr.hasStorageType()) { // If storage type (using clause) is specified
createTableNode.setStorageType(CatalogUtil.getStoreType(expr.getStorageType()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
index ef21dc3..45b5f44 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
@@ -19,12 +19,15 @@
package org.apache.tajo.engine.parser;
import com.google.common.base.Preconditions;
+
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.engine.parser.SQLParser.SqlContext;
+import org.apache.tajo.algebra.CreateTable;
import org.apache.tajo.algebra.Expr;
+import org.apache.tajo.algebra.OpType;
import org.apache.tajo.util.FileUtil;
import org.junit.Test;
@@ -268,4 +271,14 @@ public class TestHiveQLAnalyzer {
public void testDrop() throws IOException {
compareJsonResult("drop_table.sql");
}
+
+ @Test
+ public void testCreateTableLike1() throws IOException {
+ String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql"));
+ Expr expr = parseQuery(sql);
+ assertEquals(OpType.CreateTable, expr.getType());
+ CreateTable createTable = (CreateTable) expr;
+ assertEquals("orig_name", createTable.getLikeParentTableName());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
index 2010502..5423813 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
@@ -270,6 +270,15 @@ public class TestSQLAnalyzer {
}
@Test
+ public void testCreateTableLike1() throws IOException {
+ String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql"));
+ Expr expr = parseQuery(sql);
+ assertEquals(OpType.CreateTable, expr.getType());
+ CreateTable createTable = (CreateTable) expr;
+ assertEquals("orig_name", createTable.getLikeParentTableName());
+ }
+
+ @Test
public void testCreateTablePartitionByHash1() throws IOException {
String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_hash_1.sql"));
Expr expr = parseQuery(sql);
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index 2d289ba..0655700 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -23,9 +23,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.tajo.IntegrationTest;
import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.storage.StorageUtil;
+import org.apache.tajo.util.KeyValueSet;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -34,6 +39,7 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
@Category(IntegrationTest.class)
public class TestCreateTable extends QueryTestCaseBase {
@@ -357,4 +363,174 @@ public class TestCreateTable extends QueryTestCaseBase {
createdNames = executeDDL("table1_ddl.sql", "table1", "varchar");
assertTableExists(createdNames.get(0));
}
+
+ private boolean isClonedSchema(Schema origSchema, Schema newSchema) {
+ // Check schema of tables
+ boolean schemaEqual =
+ (origSchema.size() == newSchema.size());
+ if(schemaEqual == false) {
+ fail("Number of columns in schema not equal");
+ return false;
+ }
+
+ for(int col = 0; col < origSchema.size(); col++) {
+ Column colA = origSchema.getColumn(col);
+ Column colB = newSchema.getColumn(col);
+ if(colA.getSimpleName().equals(colB.getSimpleName()) == false) {
+ fail("Column names at index " + col + " do not match");
+ return false;
+ }
+ if(colA.getDataType().equals(colB.getDataType()) == false) {
+ fail("Column datatypes at index " + col + " do not match");
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean isClonedTable(String orignalTable, String newTable) throws Exception {
+ assertTableExists(newTable);
+ TableDesc origTableDesc = client.getTableDesc(orignalTable);
+ TableDesc newTableDesc = client.getTableDesc(newTable);
+
+ if(isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) {
+ fail("Schema of input tables do not match");
+ return false;
+ }
+
+ // Check partition information
+ PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod();
+ PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod();
+ if(origPartMethod != null) {
+ if(newPartMethod == null) {
+ fail("New table does not have partition info");
+ return false;
+ }
+ if(isClonedSchema(origPartMethod.getExpressionSchema(),
+ newPartMethod.getExpressionSchema()) == false) {
+ fail("Partition columns of input tables do not match");
+ return false;
+ }
+
+ if(origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false) {
+ fail("Partition type of input tables do not match");
+ return false;
+ }
+ }
+
+ // Check external flag
+ if(origTableDesc.isExternal() != newTableDesc.isExternal()) {
+ fail("External table flag on input tables not equal");
+ return false;
+ }
+
+ if(origTableDesc.getMeta() != null) {
+ TableMeta origMeta = origTableDesc.getMeta();
+ TableMeta newMeta = newTableDesc.getMeta();
+ if(origMeta.getStoreType().equals(newMeta.getStoreType()) == false) {
+ fail("Store type of input tables not equal");
+ return false;
+ }
+
+ KeyValueSet origOptions = origMeta.getOptions();
+ KeyValueSet newOptions = newMeta.getOptions();
+ if(origOptions.equals(newOptions) == false) {
+ fail("Meta options of input tables not equal");
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Test
+ public final void testCreateTableLike1() throws Exception {
+ // Basic create table with default database
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ String testMsg = "testCreateTableLike1: Basic create table with default db";
+ assertTrue(testMsg,isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+ // Basic create table with database
+ executeString("CREATE DATABASE d1").close();
+ executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close();
+ executeString("CREATE TABLE d1.table2 LIKE d1.table1");
+ testMsg = "testCreateTableLike1: Basic create table with db test failed";
+ assertTrue(testMsg, isClonedTable("d1.table1","d1.table2"));
+ executeString("DROP TABLE d1.table1");
+ executeString("DROP TABLE d1.table2");
+
+ // Table with non-default store type
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with non-default store type test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+ // Table with non-default meta options
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+
+ // Table with partitions (default partition type)
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with partitions test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+
+ // Table with external flag
+ // Use existing file as input for creating external table
+ String className = getClass().getSimpleName();
+ Path currentDatasetPath = new Path(datasetBasePath, className);
+ Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
+ executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '" + filePath.toUri() + "'").close();
+ executeString("CREATE TABLE table2 LIKE table3");
+ testMsg = "testCreateTableLike1: Table with external table flag test failed";
+ assertTrue(testMsg, isClonedTable("table3","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table2");
+
+
+ // Table created using CTAS
+ executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
+ executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2,c3 from table3;").close();
+ executeString("CREATE TABLE table2 LIKE table4");
+ testMsg = "testCreateTableLike1: Table using CTAS test failed";
+ assertTrue(testMsg, isClonedTable("table4","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table4");
+ executeString("DROP TABLE table2");
+
+
+ /* Enable when view is supported
+ // View
+ executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
+ executeString("CREATE VIEW table4(c1,c2,c3) AS SELECT c1*c1, c2, c2,c3 from table3;").close();
+ executeString("CREATE TABLE table2 LIKE table4");
+ testMsg = "testCreateTableLike1: Table using VIEW test failed";
+ assertTrue(testMsg, isClonedTable("table4","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table4");
+ executeString("DROP TABLE table2");
+ */
+
+ /* Enable when partition type other than column is supported
+ // Table with partitions (range partition)
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY RANGE (c1) ( PARTITION c1 VALUES LESS THAN (2), PARTITION c1 VALUES LESS THAN (5), PARTITION c1 VALUES LESS THAN (MAXVALUE) );").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table using non-default partition type failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+ */
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/resources/queries/default/create_table_like_1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/default/create_table_like_1.sql b/tajo-core/src/test/resources/queries/default/create_table_like_1.sql
new file mode 100644
index 0000000..73f2f46
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/default/create_table_like_1.sql
@@ -0,0 +1 @@
+create table new_table like orig_name;