You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ve...@apache.org on 2015/04/15 20:20:29 UTC
[1/3] drill git commit: DRILL-2606: Fix BIT (de)serialization issues
in ExprParser.
Repository: drill
Updated Branches:
refs/heads/master 314e5a2a8 -> 07cca5dba
DRILL-2606: Fix BIT (de)serialization issues in ExprParser.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/f55e53a1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/f55e53a1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/f55e53a1
Branch: refs/heads/master
Commit: f55e53a161857fbad92b20b382af56f4934a65c5
Parents: 09e752e
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Apr 14 22:21:39 2015 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Wed Apr 15 09:00:23 2015 -0700
----------------------------------------------------------------------
.../drill/common/expression/parser/ExprLexer.g | 3 +-
.../drill/common/expression/parser/ExprParser.g | 7 ++-
.../common/expression/parser/TreeTest.java | 5 ++
.../drill/exec/expr/TestLogicalExprSerDe.java | 55 ++++++++++++++++++++
4 files changed, 68 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/f55e53a1/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
index 092d82f..604b397 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
@@ -41,7 +41,8 @@ Nullable: 'nullable';
Repeat: 'repeat';
As: 'as';
-INT : 'int' | 'INT';
+BIT : 'bit' | 'BIT';
+INT : 'int' | 'INT';
BIGINT : 'bigint' | 'BIGINT';
FLOAT4 : 'float4' | 'FLOAT4';
FLOAT8 : 'float8' | 'FLOAT8';
http://git-wip-us.apache.org/repos/asf/drill/blob/f55e53a1/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
index 4948a02..600b791 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
@@ -101,8 +101,13 @@ dataType returns [MajorType type]
: numType {$type =$numType.type;}
| charType {$type =$charType.type;}
| dateType {$type =$dateType.type;}
+ | booleanType {$type =$booleanType.type;}
;
-
+
+booleanType returns [MajorType type]
+ : BIT { $type = Types.required(TypeProtos.MinorType.BIT); }
+ ;
+
numType returns [MajorType type]
: INT { $type = Types.required(TypeProtos.MinorType.INT); }
| BIGINT { $type = Types.required(TypeProtos.MinorType.BIGINT); }
http://git-wip-us.apache.org/repos/asf/drill/blob/f55e53a1/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
index 39aed44..7cf1477 100644
--- a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -65,6 +65,11 @@ public class TreeTest extends DrillTest {
testExpressionParsing("goodbye[4].`hello`");
}
+ @Test // DRILL-2606
+ public void testCastToBooleanExpr() throws Exception{
+ testExpressionParsing("cast( (cast( (`bool_col` ) as VARCHAR(100) ) ) as BIT )");
+ }
+
private LogicalExpression parseExpression(String expr) throws RecognitionException, IOException{
ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
http://git-wip-us.apache.org/repos/asf/drill/blob/f55e53a1/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
new file mode 100644
index 0000000..420fc75
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
@@ -0,0 +1,55 @@
+package org.apache.drill.exec.expr;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.ExecConstants;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test LogicalExpressions are serialized and deserialized properly when query is planned into multiple fragments.
+ */
+public class TestLogicalExprSerDe extends BaseTestQuery {
+
+ @BeforeClass
+ public static void setSliceCount() throws Exception {
+ // Set the slice count to low, so that query is divided into multiple fragments with exchanges.
+ test("ALTER SESSION SET `planner.slice_target`=1;");
+ }
+
+ @Test // DRILL-2606
+ public void castToBit() throws Exception {
+ testBuilder()
+ .sqlQuery("SELECT CAST(CAST('true' as VARCHAR(20)) AS BOOLEAN) c1 FROM cp.`region.json` ORDER BY `region_id` LIMIT 1")
+ .unOrdered()
+ .baselineColumns("c1")
+ .baselineValues(true)
+ .go();
+ }
+
+ // TODO: Need to add better coverage for ExprParser
+
+ @AfterClass
+ public static void resetSliceCount() throws Exception {
+ // Set the slice count to low, so that query is divided into multiple fragments with exchanges.
+ test(String.format("ALTER SESSION SET `planner.slice_target`=%d;", ExecConstants.SLICE_TARGET_DEFAULT));
+ }
+}
[3/3] drill git commit: DRILL-2422: Before creating view make sure no
table already exists with given name.
Posted by ve...@apache.org.
DRILL-2422: Before creating view make sure no table already exists with given name.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/07cca5db
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/07cca5db
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/07cca5db
Branch: refs/heads/master
Commit: 07cca5dba2b19450c35b770aa9da0e9c62fc35f8
Parents: f55e53a
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Apr 15 07:24:31 2015 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Wed Apr 15 09:00:24 2015 -0700
----------------------------------------------------------------------
.../sql/handlers/CreateTableHandler.java | 15 ++----
.../planner/sql/handlers/SqlHandlerUtil.java | 14 ++++++
.../exec/planner/sql/handlers/ViewHandler.java | 30 ++++++++----
.../physical/impl/writer/TestParquetWriter.java | 48 +++++++++++++++++++-
.../apache/drill/exec/sql/TestViewSupport.java | 40 +++++++++++++++-
5 files changed, 126 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/07cca5db/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
index a47cb47..a17a604 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
@@ -18,14 +18,11 @@
package org.apache.drill.exec.planner.sql.handlers;
import java.io.IOException;
-import java.util.List;
import net.hydromatic.optiq.SchemaPlus;
-import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
import net.hydromatic.optiq.tools.ValidationException;
-import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.planner.logical.DrillRel;
@@ -36,14 +33,10 @@ import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.sql.DrillSqlWorker;
import org.apache.drill.exec.planner.sql.parser.SqlCreateTable;
-import org.apache.drill.exec.planner.types.DrillFixedRelDataTypeImpl;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.util.Pointer;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptUtil;
-import org.eigenbase.relopt.hep.HepPlanner;
-import org.eigenbase.reltype.RelDataType;
import org.eigenbase.sql.SqlNode;
public class CreateTableHandler extends DefaultSqlHandler {
@@ -69,9 +62,11 @@ public class CreateTableHandler extends DefaultSqlHandler {
"Can't create tables in this schema.", drillSchema.getFullSchemaName()));
}
- String newTblName = sqlCreateTable.getName();
- if (schema.getTable(newTblName) != null) {
- return DirectPlan.createDirectPlan(context, false, String.format("Table '%s' already exists.", newTblName));
+ final String newTblName = sqlCreateTable.getName();
+ if (SqlHandlerUtil.getTableFromSchema(drillSchema, newTblName) != null) {
+ throw new ValidationException(
+ String.format("A table or view with given name [%s] already exists in schema [%s]",
+ newTblName, drillSchema.getFullSchemaName()));
}
log("Optiq Logical", newTblRelNode);
http://git-wip-us.apache.org/repos/asf/drill/blob/07cca5db/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
index 2f6a56d..c347bef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
@@ -17,12 +17,15 @@
*/
package org.apache.drill.exec.planner.sql.handlers;
+import net.hydromatic.optiq.Table;
import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
import net.hydromatic.optiq.tools.ValidationException;
+import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.exec.planner.common.DrillRelOptUtil;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.types.DrillFixedRelDataTypeImpl;
+import org.apache.drill.exec.store.AbstractSchema;
import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.reltype.RelDataType;
@@ -83,4 +86,15 @@ public class SqlHandlerUtil {
return validatedQueryRelNode;
}
+
+ public static Table getTableFromSchema(AbstractSchema drillSchema, String tblName) throws DrillException {
+ try {
+ return drillSchema.getTable(tblName);
+ } catch (Exception e) {
+ // TODO: Move to better exception types.
+ throw new DrillException(
+ String.format("Failure while trying to check if a table or view with given name [%s] already exists " +
+ "in schema [%s]: %s", tblName, drillSchema.getFullSchemaName(), e.getMessage()), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/07cca5db/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
index 5c0c2f2..00fc522 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
@@ -20,7 +20,9 @@ package org.apache.drill.exec.planner.sql.handlers;
import java.io.IOException;
import java.util.List;
+import net.hydromatic.optiq.Schema.TableType;
import net.hydromatic.optiq.SchemaPlus;
+import net.hydromatic.optiq.Table;
import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
import net.hydromatic.optiq.tools.ValidationException;
@@ -28,18 +30,13 @@ import net.hydromatic.optiq.tools.ValidationException;
import org.apache.drill.exec.dotdrill.View;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
-import org.apache.drill.exec.planner.common.DrillRelOptUtil;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.sql.parser.SqlCreateView;
import org.apache.drill.exec.planner.sql.parser.SqlDropView;
-import org.apache.drill.exec.planner.types.DrillFixedRelDataTypeImpl;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
-import org.eigenbase.rel.CalcRel;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptUtil;
-import org.eigenbase.reltype.RelDataType;
import org.eigenbase.sql.SqlNode;
import com.google.common.collect.ImmutableList;
@@ -91,12 +88,27 @@ public abstract class ViewHandler extends AbstractSqlHandler {
View view = new View(createView.getName(), viewSql, newViewRelNode.getRowType(), workspaceSchemaPath);
+ final String viewName = view.getName();
+ final Table existingTable = SqlHandlerUtil.getTableFromSchema(drillSchema, viewName);
+
+ if (existingTable != null) {
+ if (existingTable.getJdbcTableType() != TableType.VIEW) {
+ // existing table is not a view
+ throw new ValidationException(
+ String.format("A non-view table with given name [%s] already exists in schema [%s]",
+ viewName, schemaPath));
+ }
+
+ if (existingTable.getJdbcTableType() == TableType.VIEW && !createView.getReplace()) {
+ // existing table is a view and create view has no "REPLACE" clause
+ throw new ValidationException(
+ String.format("A view with given name [%s] already exists in schema [%s]",
+ view.getName(), schemaPath));
+ }
+ }
+
boolean replaced;
if (drillSchema instanceof WorkspaceSchema) {
- WorkspaceSchema workspaceSchema = (WorkspaceSchema) drillSchema;
- if (!createView.getReplace() && workspaceSchema.viewExists(view.getName())) {
- return DirectPlan.createDirectPlan(context, false, "View with given name already exists in current schema");
- }
replaced = ((WorkspaceSchema) drillSchema).createView(view);
} else {
return DirectPlan.createDirectPlan(context, false, "Schema provided was not a workspace schema.");
http://git-wip-us.apache.org/repos/asf/drill/blob/07cca5db/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index df6e43d..89837e7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -342,7 +342,7 @@ public class TestParquetWriter extends BaseTestQuery {
@Ignore
@Test
public void test958_sql_all_columns() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/store_sales`");
+ compareParquetReadersHyperVector("*", "dfs.`/tmp/store_sales`");
compareParquetReadersHyperVector("ss_addr_sk, ss_hdemo_sk", "dfs.`/tmp/store_sales`");
// TODO - Drill 1388 - this currently fails, but it is an issue with project, not the reader, pulled out the physical plan
// removed the unneeded project in the plan and ran it against both readers, they outputs matched
@@ -447,6 +447,52 @@ public class TestParquetWriter extends BaseTestQuery {
}
}
+ @Test // DRILL-2422
+ public void createTableWhenATableWithSameNameAlreadyExists() throws Exception{
+ final String newTblName = "createTableWhenTableAlreadyExists";
+
+ try {
+ test("USE dfs_test.tmp");
+ final String ctas = String.format("CREATE TABLE %s AS SELECT * from cp.`region.json`", newTblName);
+
+ test(ctas);
+
+ testBuilder()
+ .unOrdered()
+ .sqlQuery(ctas)
+ .baselineColumns("ok", "summary")
+ .baselineValues(false,
+ String.format("Error: A table or view with given name [%s] already exists in schema [%s]",
+ newTblName, "dfs_test.tmp"))
+ .go();
+ } finally {
+ deleteTableIfExists(newTblName);
+ }
+ }
+
+ @Test // DRILL-2422
+ public void createTableWhenAViewWithSameNameAlreadyExists() throws Exception{
+ final String newTblName = "createTableWhenAViewWithSameNameAlreadyExists";
+
+ try {
+ test("USE dfs_test.tmp");
+ final String createView = String.format("CREATE VIEW %s AS SELECT * from cp.`region.json`", newTblName);
+
+ test(createView);
+
+ testBuilder()
+ .unOrdered()
+ .sqlQuery(String.format("CREATE TABLE %s AS SELECT * FROM cp.`employee.json`", newTblName))
+ .baselineColumns("ok", "summary")
+ .baselineValues(false,
+ String.format("Error: A table or view with given name [%s] already exists in schema [%s]",
+ newTblName, "dfs_test.tmp"))
+ .go();
+ } finally {
+ test("DROP VIEW " + newTblName);
+ }
+ }
+
@Test // see DRILL-2408
public void testWriteEmptyFileAfterFlush() throws Exception {
String outputFile = "testparquetwriter_test_write_empty_file_after_flush";
http://git-wip-us.apache.org/repos/asf/drill/blob/07cca5db/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
index b0b2bb8..2ae6991 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
@@ -18,9 +18,11 @@
package org.apache.drill.exec.sql;
import com.google.common.collect.ImmutableList;
+import org.apache.commons.io.FileUtils;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.File;
import java.util.List;
public class TestViewSupport extends TestBaseViewSupport {
@@ -275,7 +277,8 @@ public class TestViewSupport extends TestBaseViewSupport {
.sqlQuery(createViewSql)
.unOrdered()
.baselineColumns("ok", "summary")
- .baselineValues(false, "View with given name already exists in current schema")
+ .baselineValues(false,
+ String.format("Error: A view with given name [%s] already exists in schema [%s]", viewName, TEMP_SCHEMA))
.go();
// Try creating the view with same name in same schema, but with CREATE OR REPLACE VIEW clause
@@ -301,6 +304,41 @@ public class TestViewSupport extends TestBaseViewSupport {
}
}
+ @Test // DRILL-2422
+ public void createViewWhenATableWithSameNameAlreadyExists() throws Exception {
+ final String tableName = generateViewName();
+
+ try {
+ final String tableDef1 = "SELECT region_id, sales_city FROM cp.`region.json`";
+
+ test(String.format("CREATE TABLE %s.%s as %s", TEMP_SCHEMA, tableName, tableDef1));
+
+ // Try to create the view with same name in same schema.
+ final String createViewSql = String.format("CREATE VIEW %s.`%s` AS %s", TEMP_SCHEMA, tableName, tableDef1);
+ testBuilder()
+ .sqlQuery(createViewSql)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(false,
+ String.format("Error: A non-view table with given name [%s] already exists in schema [%s]",
+ tableName, TEMP_SCHEMA))
+ .go();
+
+ // Try creating the view with same name in same schema, but with CREATE OR REPLACE VIEW clause
+ final String viewDef2 = "SELECT sales_state_province FROM cp.`region.json` ORDER BY `region_id`";
+ testBuilder()
+ .sqlQuery(String.format("CREATE OR REPLACE VIEW %s.`%s` AS %s", TEMP_SCHEMA, tableName, viewDef2))
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(false,
+ String.format("Error: A non-view table with given name [%s] already exists in schema [%s]",
+ tableName, TEMP_SCHEMA))
+ .go();
+ } finally {
+ FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), tableName));
+ }
+ }
+
@Test
public void infoSchemaWithView() throws Exception {
final String viewName = generateViewName();
[2/3] drill git commit: DRILL-2341: Get type information from view
def when views field list is specified.
Posted by ve...@apache.org.
DRILL-2341: Get type information from view def when views field list is specified.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/09e752ee
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/09e752ee
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/09e752ee
Branch: refs/heads/master
Commit: 09e752ee14bbe2f1433a05a9ca9390f575f1232f
Parents: 314e5a2
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Apr 14 19:28:44 2015 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Wed Apr 15 09:00:23 2015 -0700
----------------------------------------------------------------------
.../exec/planner/common/DrillRelOptUtil.java | 36 ++++++++
.../sql/handlers/CreateTableHandler.java | 38 +--------
.../planner/sql/handlers/SqlHandlerUtil.java | 86 ++++++++++++++++++++
.../exec/planner/sql/handlers/ViewHandler.java | 38 +++------
.../physical/impl/writer/TestParquetWriter.java | 26 ++++++
.../apache/drill/exec/sql/TestViewSupport.java | 29 +++++++
6 files changed, 191 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/09e752ee/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
index cacf26b..bbe7cf3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.common;
+import java.util.AbstractList;
import java.util.List;
import com.google.common.collect.Lists;
@@ -24,8 +25,12 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.planner.logical.DrillOptiq;
import org.apache.drill.exec.resolver.TypeCastRules;
+import org.eigenbase.rel.CalcRel;
+import org.eigenbase.rel.RelNode;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeField;
+import org.eigenbase.rex.RexInputRef;
+import org.eigenbase.rex.RexNode;
import org.eigenbase.sql.type.SqlTypeName;
import org.eigenbase.util.Pair;
@@ -82,4 +87,35 @@ public abstract class DrillRelOptUtil {
}
return true;
}
+
+ /**
+ * Returns a relational expression which has the same fields as the
+ * underlying expression, but the fields have different names.
+ *
+ * Note: This method is copied from {@link org.eigenbase.rel.CalcRel#createRename(RelNode, List)} because it has a bug
+ * which doesn't rename the exprs. This bug is fixed in latest version of Apache Calcite (1.2).
+ *
+ * @param rel Relational expression
+ * @param fieldNames Field names
+ * @return Renamed relational expression
+ */
+ public static RelNode createRename(
+ RelNode rel,
+ final List<String> fieldNames) {
+ final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
+ assert fieldNames.size() == fields.size();
+ final List<Pair<RexNode, String>> refs =
+ new AbstractList<Pair<RexNode, String>>() {
+ public int size() {
+ return fields.size();
+ }
+
+ public Pair<RexNode, String> get(int index) {
+ return Pair.of(
+ (RexNode) new RexInputRef(index, fields.get(index).getType()),
+ fieldNames.get(index));
+ }
+ };
+ return CalcRel.createProject(rel, refs, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/09e752ee/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
index 300d65d..a47cb47 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateTableHandler.java
@@ -56,38 +56,8 @@ public class CreateTableHandler extends DefaultSqlHandler {
SqlCreateTable sqlCreateTable = unwrap(sqlNode, SqlCreateTable.class);
try {
- // Convert the query in CTAS statement into a RelNode
- SqlNode validatedQuery = validateNode(sqlCreateTable.getQuery());
- RelNode relQuery = convertToRel(validatedQuery);
-
- List<String> tblFiledNames = sqlCreateTable.getFieldNames();
- RelDataType queryRowType = relQuery.getRowType();
-
- if (tblFiledNames.size() > 0) {
- // Field count should match.
- if (tblFiledNames.size() != queryRowType.getFieldCount()) {
- return DirectPlan.createDirectPlan(context, false,
- "Table's field list and the table's query field list have different counts.");
- }
-
- // CTAS's query field list shouldn't have "*" when table's field list is specified.
- for (String field : queryRowType.getFieldNames()) {
- if (field.equals("*")) {
- return DirectPlan.createDirectPlan(context, false,
- "Table's query field list has a '*', which is invalid when table's field list is specified.");
- }
- }
- }
-
- // if the CTAS statement has table fields lists (ex. below), add a project rel to rename the query fields.
- // Ex. CREATE TABLE tblname(col1, medianOfCol2, avgOfCol3) AS
- // SELECT col1, median(col2), avg(col3) FROM sourcetbl GROUP BY col1 ;
- if (tblFiledNames.size() > 0) {
- // create rowtype to which the select rel needs to be casted.
- RelDataType rowType = new DrillFixedRelDataTypeImpl(planner.getTypeFactory(), tblFiledNames);
-
- relQuery = RelOptUtil.createCastRel(relQuery, rowType, true);
- }
+ final RelNode newTblRelNode =
+ SqlHandlerUtil.resolveNewTableRel(false, planner, sqlCreateTable.getFieldNames(), sqlCreateTable.getQuery());
SchemaPlus schema = findSchema(context.getRootSchema(), context.getNewDefaultSchema(),
sqlCreateTable.getSchemaPath());
@@ -104,10 +74,10 @@ public class CreateTableHandler extends DefaultSqlHandler {
return DirectPlan.createDirectPlan(context, false, String.format("Table '%s' already exists.", newTblName));
}
- log("Optiq Logical", relQuery);
+ log("Optiq Logical", newTblRelNode);
// Convert the query to Drill Logical plan and insert a writer operator on top.
- DrillRel drel = convertToDrel(relQuery, drillSchema, newTblName);
+ DrillRel drel = convertToDrel(newTblRelNode, drillSchema, newTblName);
log("Drill Logical", drel);
Prel prel = convertToPrel(drel);
log("Drill Physical", prel);
http://git-wip-us.apache.org/repos/asf/drill/blob/09e752ee/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
new file mode 100644
index 0000000..2f6a56d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.handlers;
+
+import net.hydromatic.optiq.tools.Planner;
+import net.hydromatic.optiq.tools.RelConversionException;
+import net.hydromatic.optiq.tools.ValidationException;
+import org.apache.drill.exec.planner.common.DrillRelOptUtil;
+import org.apache.drill.exec.planner.sql.DirectPlan;
+import org.apache.drill.exec.planner.types.DrillFixedRelDataTypeImpl;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptUtil;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.sql.SqlNode;
+
+import java.util.List;
+
+public class SqlHandlerUtil {
+
+ /**
+ * Resolve final RelNode of the new table (or view) for given table field list and new table definition.
+ *
+ * @param isNewTableView Is the new table created a view? This doesn't affect the functionality, but it helps format
+ * better error messages.
+ * @param planner Planner instance.
+ * @param tableFieldNames List of fields specified in new table/view field list. These are the fields given just after
+ * new table name.
+ * Ex. CREATE TABLE newTblName(col1, medianOfCol2, avgOfCol3) AS
+ * SELECT col1, median(col2), avg(col3) FROM sourcetbl GROUP BY col1;
+ * @param newTableQueryDef Sql tree of definition of the new table or view (query after the AS keyword). This tree is
+ * modified, so it is responsibility of caller's to make a copy if needed.
+ * @throws ValidationException If table's fields list and field list specified in table definition are not valid.
+ * @throws RelConversionException If failed to convert the table definition into a RelNode.
+ */
+ public static RelNode resolveNewTableRel(boolean isNewTableView, Planner planner, List<String> tableFieldNames,
+ SqlNode newTableQueryDef) throws ValidationException, RelConversionException {
+
+ SqlNode validatedQuery = planner.validate(newTableQueryDef);
+ RelNode validatedQueryRelNode = planner.convert(validatedQuery);
+
+ if (tableFieldNames.size() > 0) {
+ final RelDataType queryRowType = validatedQueryRelNode.getRowType();
+
+ // Field count should match.
+ if (tableFieldNames.size() != queryRowType.getFieldCount()) {
+ final String tblType = isNewTableView ? "view" : "table";
+ throw new ValidationException(
+ String.format("%s's field list and the %s's query field list have different counts.", tblType, tblType));
+ }
+
+ // CTAS's query field list shouldn't have "*" when table's field list is specified.
+ for (String field : queryRowType.getFieldNames()) {
+ if (field.equals("*")) {
+ final String tblType = isNewTableView ? "view" : "table";
+ throw new ValidationException(
+ String.format("%s's query field list has a '*', which is invalid when %s's field list is specified.",
+ tblType, tblType));
+ }
+ }
+
+ // CTAS statement has table field list (ex. below), add a project rel to rename the query fields.
+ // Ex. CREATE TABLE tblname(col1, medianOfCol2, avgOfCol3) AS
+ // SELECT col1, median(col2), avg(col3) FROM sourcetbl GROUP BY col1 ;
+ // Similary for CREATE VIEW.
+
+ return DrillRelOptUtil.createRename(validatedQueryRelNode, tableFieldNames);
+ }
+
+ return validatedQueryRelNode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/09e752ee/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
index 4347249..5c0c2f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
@@ -28,6 +28,7 @@ import net.hydromatic.optiq.tools.ValidationException;
import org.apache.drill.exec.dotdrill.View;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.planner.common.DrillRelOptUtil;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.sql.parser.SqlCreateView;
import org.apache.drill.exec.planner.sql.parser.SqlDropView;
@@ -35,7 +36,9 @@ import org.apache.drill.exec.planner.types.DrillFixedRelDataTypeImpl;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
+import org.eigenbase.rel.CalcRel;
import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.sql.SqlNode;
@@ -64,6 +67,12 @@ public abstract class ViewHandler extends AbstractSqlHandler {
SqlCreateView createView = unwrap(sqlNode, SqlCreateView.class);
try {
+ // Store the viewSql as view def SqlNode is modified as part of the resolving the new table definition below.
+ final String viewSql = createView.getQuery().toString();
+
+ final RelNode newViewRelNode =
+ SqlHandlerUtil.resolveNewTableRel(true, planner, createView.getFieldNames(), createView.getQuery());
+
SchemaPlus defaultSchema = context.getNewDefaultSchema();
SchemaPlus schema = findSchema(context.getRootSchema(), defaultSchema, createView.getSchemaPath());
AbstractSchema drillSchema = getDrillSchema(schema);
@@ -80,34 +89,7 @@ public abstract class ViewHandler extends AbstractSqlHandler {
workspaceSchemaPath = getDrillSchema(defaultSchema).getSchemaPath();
}
- String viewSql = createView.getQuery().toString();
-
- SqlNode validatedQuery = planner.validate(createView.getQuery());
- RelNode validatedRelNode = planner.convert(validatedQuery);
-
- // If view's field list is specified then its size should match view's query field list size.
- RelDataType queryRowType = validatedRelNode.getRowType();
-
- List<String> viewFieldNames = createView.getFieldNames();
- if (viewFieldNames.size() > 0) {
- // number of fields match.
- if (viewFieldNames.size() != queryRowType.getFieldCount()) {
- return DirectPlan.createDirectPlan(context, false,
- "View's field list and View's query field list have different counts.");
- }
-
- // make sure View's query field list has no "*"
- for (String field : queryRowType.getFieldNames()) {
- if (field.equals("*")) {
- return DirectPlan.createDirectPlan(context, false,
- "View's query field list has a '*', which is invalid when View's field list is specified.");
- }
- }
-
- queryRowType = new DrillFixedRelDataTypeImpl(planner.getTypeFactory(), viewFieldNames);
- }
-
- View view = new View(createView.getName(), viewSql, queryRowType, workspaceSchemaPath);
+ View view = new View(createView.getName(), viewSql, newViewRelNode.getRowType(), workspaceSchemaPath);
boolean replaced;
if (drillSchema instanceof WorkspaceSchema) {
http://git-wip-us.apache.org/repos/asf/drill/blob/09e752ee/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 325f770..df6e43d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -19,12 +19,14 @@ package org.apache.drill.exec.physical.impl.writer;
import java.io.File;
import java.math.BigDecimal;
+import java.sql.Date;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.exec.ExecConstants;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -421,6 +423,30 @@ public class TestParquetWriter extends BaseTestQuery {
}
}
+ @Test // DRILL-2341
+ public void tableSchemaWhenSelectFieldsInDef_SelectFieldsInView() throws Exception {
+ final String newTblName = "testTableOutputSchema";
+
+ try {
+ final String ctas = String.format("CREATE TABLE dfs_test.tmp.%s(id, name, bday) AS SELECT " +
+ "cast(`employee_id` as integer), " +
+ "cast(`full_name` as varchar(100)), " +
+ "cast(`birth_date` as date) " +
+ "FROM cp.`employee.json` ORDER BY `employee_id` LIMIT 1", newTblName);
+
+ test(ctas);
+
+ testBuilder()
+ .unOrdered()
+ .sqlQuery(String.format("SELECT * FROM dfs_test.tmp.`%s`", newTblName))
+ .baselineColumns("id", "name", "bday")
+ .baselineValues(1, "Sheri Nowmer", new DateTime(Date.valueOf("1961-08-26").getTime()))
+ .go();
+ } finally {
+ deleteTableIfExists(newTblName);
+ }
+ }
+
@Test // see DRILL-2408
public void testWriteEmptyFileAfterFlush() throws Exception {
String outputFile = "testparquetwriter_test_write_empty_file_after_flush";
http://git-wip-us.apache.org/repos/asf/drill/blob/09e752ee/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
index 578eace..b0b2bb8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
@@ -413,4 +413,33 @@ public class TestViewSupport extends TestBaseViewSupport {
dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
}
}
+
+ // DRILL-2341, View schema verification where view's field is not specified is already tested in
+ // TestViewSupport.infoSchemaWithView.
+ @Test
+ public void viewSchemaWhenSelectFieldsInDef_SelectFieldsInView() throws Exception {
+ final String viewName = generateViewName();
+
+ try {
+ test("USE " + TEMP_SCHEMA);
+ createViewHelper(null, viewName, TEMP_SCHEMA, "(id, name, bday)",
+ "SELECT " +
+ "cast(`region_id` as integer), " +
+ "cast(`full_name` as varchar(100)), " +
+ "cast(`birth_date` as date) " +
+ "FROM cp.`employee.json`");
+
+ // Test DESCRIBE view
+ testBuilder()
+ .sqlQuery(String.format("DESCRIBE `%s`", viewName))
+ .unOrdered()
+ .baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
+ .baselineValues("id", "INTEGER", "YES")
+ .baselineValues("name", "VARCHAR", "YES")
+ .baselineValues("bday", "DATE", "YES")
+ .go();
+ } finally {
+ dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
+ }
+ }
}