You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2019/03/29 14:47:44 UTC
[drill] 01/03: DRILL-7138: Implement command to describe schema for
table
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5f5b3ca6a8af175e603877689c4b15a3126fe016
Author: Arina Ielchiieva <ar...@gmail.com>
AuthorDate: Thu Mar 28 18:04:31 2019 +0200
DRILL-7138: Implement command to describe schema for table
closes #1719
---
.../src/main/codegen/includes/parserImpls.ftl | 39 +++++--
.../drill/exec/planner/sql/DrillSqlWorker.java | 7 ++
.../exec/planner/sql/handlers/SchemaHandler.java | 95 ++++++++++++++++-
.../sql/parser/CompoundIdentifierConverter.java | 1 +
.../drill/exec/planner/sql/parser/SqlSchema.java | 64 +++++++++++-
.../record/metadata/schema/PathSchemaProvider.java | 4 +-
.../java/org/apache/drill/TestSchemaCommands.java | 112 +++++++++++++++++++++
7 files changed, 311 insertions(+), 11 deletions(-)
diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
index 015ba94..97b5052 100644
--- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
+++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
@@ -179,7 +179,7 @@ SqlNodeList ParseRequiredFieldList(String relType) :
}
/**
-* Rarses CREATE [OR REPLACE] command for VIEW, TABLE or SCHEMA.
+* Parses CREATE [OR REPLACE] command for VIEW, TABLE or SCHEMA.
*/
SqlNode SqlCreateOrReplace() :
{
@@ -374,7 +374,7 @@ void addProperty(SqlNodeList properties) :
<SCH> TOKEN : {
< LOAD: "LOAD" > { popState(); }
| < NUM: <DIGIT> (" " | "\t" | "\n" | "\r")* >
- // once schema is found, swich back to initial lexical state
+ // once schema is found, switch back to initial lexical state
// must be enclosed in the parentheses
// inside may have left parenthesis only if number precededs (covers cases with varchar(10)),
// if left parenthesis is present in column name, it must be escaped with backslash
@@ -494,18 +494,43 @@ SqlNode SqlRefreshMetadata() :
/**
* Parses statement
* DESCRIBE { SCHEMA | DATABASE } name
+* DESCRIBE SCHEMA FOR TABLE dfs.my_table [AS (JSON | STATEMENT)]
*/
SqlNode SqlDescribeSchema() :
{
SqlParserPos pos;
- SqlIdentifier schema;
+ SqlIdentifier table;
+ String format = "JSON";
}
{
<DESCRIBE> { pos = getPos(); }
- (<SCHEMA> | <DATABASE>) { schema = CompoundIdentifier(); }
- {
- return new SqlDescribeSchema(pos, schema);
- }
+ (
+ <SCHEMA>
+ (
+ <FOR> <TABLE> { table = CompoundIdentifier(); }
+ [
+ <AS>
+ (
+ <JSON> { format = "JSON"; }
+ |
+ <STATEMENT> { format = "STATEMENT"; }
+ )
+ ]
+ {
+ return new SqlSchema.Describe(pos, table, SqlLiteral.createCharString(format, getPos()));
+ }
+
+ |
+ {
+ return new SqlDescribeSchema(pos, CompoundIdentifier());
+ }
+ )
+ |
+ <DATABASE>
+ {
+ return new SqlDescribeSchema(pos, CompoundIdentifier());
+ }
+ )
}
/**
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 7b9050f..449dc17 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -36,11 +36,13 @@ import org.apache.drill.exec.planner.sql.handlers.DescribeSchemaHandler;
import org.apache.drill.exec.planner.sql.handlers.DescribeTableHandler;
import org.apache.drill.exec.planner.sql.handlers.ExplainHandler;
import org.apache.drill.exec.planner.sql.handlers.RefreshMetadataHandler;
+import org.apache.drill.exec.planner.sql.handlers.SchemaHandler;
import org.apache.drill.exec.planner.sql.handlers.SetOptionHandler;
import org.apache.drill.exec.planner.sql.handlers.SqlHandlerConfig;
import org.apache.drill.exec.planner.sql.parser.DrillSqlCall;
import org.apache.drill.exec.planner.sql.parser.DrillSqlDescribeTable;
import org.apache.drill.exec.planner.sql.parser.SqlCreateTable;
+import org.apache.drill.exec.planner.sql.parser.SqlSchema;
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
import org.apache.drill.exec.util.Pointer;
@@ -164,6 +166,11 @@ public class DrillSqlWorker {
context.setSQLStatementType(SqlStatementType.DESCRIBE_SCHEMA);
break;
}
+ if (sqlNode instanceof SqlSchema.Describe) {
+ handler = new SchemaHandler.Describe(config);
+ context.setSQLStatementType(SqlStatementType.DESCRIBE_SCHEMA);
+ break;
+ }
case CREATE_TABLE:
handler = ((DrillSqlCall) sqlNode).getSqlHandler(config, textPlan);
break;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
index 4882683..9df0eca 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
@@ -29,8 +29,11 @@ import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.sql.SchemaUtilites;
import org.apache.drill.exec.planner.sql.parser.SqlCreateType;
import org.apache.drill.exec.planner.sql.parser.SqlSchema;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.record.metadata.schema.FsMetastoreSchemaProvider;
import org.apache.drill.exec.record.metadata.schema.PathSchemaProvider;
+import org.apache.drill.exec.record.metadata.schema.SchemaContainer;
import org.apache.drill.exec.record.metadata.schema.SchemaProvider;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.StorageStrategy;
@@ -44,9 +47,11 @@ import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
/**
- * Parent class for CREATE / DROP SCHEMA handlers.
+ * Parent class for CREATE / DROP / DESCRIBE SCHEMA handlers.
* Contains common logic on how extract workspace, output error result.
*/
public abstract class SchemaHandler extends DefaultSqlHandler {
@@ -211,4 +216,92 @@ public abstract class SchemaHandler extends DefaultSqlHandler {
}
}
+ /**
+ * DESCRIBE SCHEMA FOR TABLE command handler.
+ */
+ public static class Describe extends SchemaHandler {
+
+ public Describe(SqlHandlerConfig config) {
+ super(config);
+ }
+
+ @Override
+ public PhysicalPlan getPlan(SqlNode sqlNode) {
+ SqlSchema.Describe sqlCall = ((SqlSchema.Describe) sqlNode);
+
+ String tableName = sqlCall.getTableName();
+ WorkspaceSchemaFactory.WorkspaceSchema wsSchema = getWorkspaceSchema(sqlCall.getSchemaPath(), tableName);
+
+ try {
+
+ SchemaProvider schemaProvider = new FsMetastoreSchemaProvider(wsSchema, tableName);
+
+ if (schemaProvider.exists()) {
+
+ SchemaContainer schemaContainer = schemaProvider.read();
+
+ String schema;
+ switch (sqlCall.getFormat()) {
+ case JSON:
+ schema = PathSchemaProvider.WRITER.writeValueAsString(schemaContainer);
+ break;
+ case STATEMENT:
+ TupleMetadata metadata = schemaContainer.getSchema();
+ StringBuilder builder = new StringBuilder("CREATE OR REPLACE SCHEMA \n");
+ builder.append("(\n");
+
+ builder.append(metadata.toMetadataList().stream()
+ .map(ColumnMetadata::columnString)
+ .collect(Collectors.joining(", \n")));
+
+ builder.append("\n) \n");
+
+ builder.append("FOR TABLE ").append(schemaContainer.getTable()).append(" \n");
+
+ Map<String, String> properties = metadata.properties();
+ if (!properties.isEmpty()) {
+ builder.append("PROPERTIES (\n");
+
+ builder.append(properties.entrySet().stream()
+ .map(e -> String.format("'%s' = '%s'", e.getKey(), e.getValue()))
+ .collect(Collectors.joining(", \n")));
+ builder.append("\n)");
+ }
+
+ schema = builder.toString();
+ break;
+ default:
+ throw UserException.validationError()
+ .message("Unsupported describe schema format: [%s]", sqlCall.getFormat())
+ .build(logger);
+ }
+
+ return DirectPlan.createDirectPlan(context, new SchemaResult(schema));
+ }
+
+ return DirectPlan.createDirectPlan(context, false,
+ String.format("Schema for table [%s] is absent", sqlCall.getTable()));
+
+ } catch (IOException e) {
+ throw UserException.resourceError(e)
+ .message(e.getMessage())
+ .addContext("Error while accessing table location for [%s]", sqlCall.getTable())
+ .build(logger);
+ }
+ }
+
+ /**
+ * Wrapper to output schema in a form of table with one column named `schema`.
+ */
+ public static class SchemaResult {
+
+ public String schema;
+
+ public SchemaResult(String schema) {
+ this.schema = schema;
+ }
+ }
+
+ }
+
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
index 5d3d080..119b27d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
@@ -82,6 +82,7 @@ public class CompoundIdentifierConverter extends SqlShuttle {
.put(SqlDropFunction.class, arrayOf(D))
.put(SqlSchema.Create.class, arrayOf(D, D, D, D, D, D))
.put(SqlSchema.Drop.class, arrayOf(D, D))
+ .put(SqlSchema.Describe.class, arrayOf(D, D))
.build();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java
index 7985279..bfbf06f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java
@@ -41,7 +41,7 @@ import java.util.List;
import java.util.Map;
/**
- * Parent class for CREATE and DROP SCHEMA commands.
+ * Parent class for CREATE, DROP, DESCRIBE SCHEMA commands.
* Holds logic common command property: table.
*/
public abstract class SqlSchema extends DrillSqlCall {
@@ -277,4 +277,66 @@ public abstract class SqlSchema extends DrillSqlCall {
}
+ /**
+ * DESCRIBE SCHEMA FOR TABLE sql call.
+ */
+ public static class Describe extends SqlSchema {
+
+ private final SqlLiteral format;
+
+ public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator(SqlKind.DESCRIBE_SCHEMA.name(), SqlKind.DESCRIBE_SCHEMA) {
+ @Override
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new Describe(pos, (SqlIdentifier) operands[0], (SqlLiteral) operands[1]);
+ }
+ };
+
+ public Describe(SqlParserPos pos, SqlIdentifier table, SqlLiteral format) {
+ super(pos, table);
+ this.format = format;
+ }
+
+ @Override
+ public SqlOperator getOperator() {
+ return OPERATOR;
+ }
+
+ @Override
+ public List<SqlNode> getOperandList() {
+ return Arrays.asList(table, format);
+ }
+
+ @Override
+ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) {
+ writer.keyword("DESCRIBE");
+ writer.keyword("SCHEMA");
+
+ super.unparse(writer, leftPrec, rightPrec);
+
+ writer.keyword("AS");
+ writer.keyword(getFormat().name());
+ }
+
+ public Describe.Format getFormat() {
+ return Format.valueOf(format.toValue());
+ }
+
+ /**
+ * Enum which specifies format of DESCRIBE SCHEMA FOR table output.
+ */
+ public enum Format {
+
+ /**
+ * Schema will be output in JSON format used to store schema
+ * in {@link org.apache.drill.exec.record.metadata.schema.SchemaProvider#DEFAULT_SCHEMA_NAME} file.
+ */
+ JSON,
+
+ /**
+ * Schema will be output in CREATE SCHEMA command syntax.
+ */
+ STATEMENT
+ }
+ }
+
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java
index 28754aa..d73e247 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java
@@ -45,12 +45,12 @@ public class PathSchemaProvider implements SchemaProvider {
* Reader used to read JSON schema from file into into {@link SchemaContainer}.
* Allows comment inside the JSON file.
*/
- private static final ObjectReader READER;
+ public static final ObjectReader READER;
/**
* Writer used to write content from {@link SchemaContainer} into JSON file.
*/
- private static final ObjectWriter WRITER;
+ public static final ObjectWriter WRITER;
static {
ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java
index 92a7b27..7391366 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java
@@ -553,4 +553,116 @@ public class TestSchemaCommands extends ClusterTest {
}
}
+ @Test
+ public void testDescribeForMissingTable() throws Exception {
+ thrown.expect(UserException.class);
+ thrown.expectMessage("VALIDATION ERROR: Table [t] was not found");
+
+ run("describe schema for table dfs.t");
+ }
+
+ @Test
+ public void testDescribeForMissingSchema() throws Exception {
+ String table = "dfs.tmp.table_describe_with_missing_schema";
+ try {
+ run("create table %s as select 'a' as c from (values(1))", table);
+
+ testBuilder()
+ .sqlQuery("describe schema for table %s", table)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(false, String.format("Schema for table [%s] is absent", table))
+ .go();
+
+ } finally {
+ run("drop table if exists %s", table);
+ }
+ }
+
+ @Test
+ public void testDescribeDefault() throws Exception {
+ String tableName = "table_describe_default";
+ String table = String.format("dfs.tmp.%s", tableName);
+ try {
+ run("create table %s as select 'a' as c from (values(1))", table);
+ run("create schema (col int) for table %s", table);
+
+ File schemaFile = Paths.get(dirTestWatcher.getDfsTestTmpDir().getPath(),
+ tableName, SchemaProvider.DEFAULT_SCHEMA_NAME).toFile();
+
+ SchemaProvider schemaProvider = new PathSchemaProvider(new Path(schemaFile.getPath()));
+ SchemaContainer schemaContainer = schemaProvider.read();
+ String schema = PathSchemaProvider.WRITER.writeValueAsString(schemaContainer);
+
+ testBuilder()
+ .sqlQuery("describe schema for table %s", table)
+ .unOrdered()
+ .baselineColumns("schema")
+ .baselineValues(schema)
+ .go();
+
+ } finally {
+ run("drop table if exists %s", table);
+ }
+ }
+
+ @Test
+ public void testDescribeJson() throws Exception {
+ String tableName = "table_describe_json";
+ String table = String.format("dfs.tmp.%s", tableName);
+ try {
+ run("create table %s as select 'a' as c from (values(1))", table);
+ run("create schema (col int) for table %s", table);
+
+ File schemaFile = Paths.get(dirTestWatcher.getDfsTestTmpDir().getPath(),
+ tableName, SchemaProvider.DEFAULT_SCHEMA_NAME).toFile();
+
+ SchemaProvider schemaProvider = new PathSchemaProvider(new Path(schemaFile.getPath()));
+ SchemaContainer schemaContainer = schemaProvider.read();
+ String schema = PathSchemaProvider.WRITER.writeValueAsString(schemaContainer);
+
+ testBuilder()
+ .sqlQuery("describe schema for table %s as json", table)
+ .unOrdered()
+ .baselineColumns("schema")
+ .baselineValues(schema)
+ .go();
+
+ } finally {
+ run("drop table if exists %s", table);
+ }
+ }
+
+ @Test
+ public void testDescribeStatement() throws Exception {
+ String tableName = "table_describe_statement";
+ String table = String.format("dfs.tmp.%s", tableName);
+ try {
+ run("create table %s as select 'a' as c from (values(1))", table);
+
+ String statement = "CREATE OR REPLACE SCHEMA \n"
+ + "(\n"
+ + "`col1` DATE FORMAT 'yyyy-MM-dd' DEFAULT '-1', \n"
+ + "`col2` INT NOT NULL FORMAT 'yyyy-MM-dd' PROPERTIES { 'drill.strict' = 'true', 'some_column_prop' = 'some_column_val' }\n"
+ + ") \n"
+ + "FOR TABLE dfs.tmp.`table_describe_statement` \n"
+ + "PROPERTIES (\n"
+ + "'drill.strict' = 'false', \n"
+ + "'some_schema_prop' = 'some_schema_val'\n"
+ + ")";
+
+ run(statement);
+
+ testBuilder()
+ .sqlQuery("describe schema for table %s as statement", table)
+ .unOrdered()
+ .baselineColumns("schema")
+ .baselineValues(statement)
+ .go();
+
+ } finally {
+ run("drop table if exists %s", table);
+ }
+ }
+
}