You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/12 18:35:06 UTC
[15/24] git commit: DRILL-858: Fix bugs in handling Hive views in
INFORMATION_SCHEMA generation.
DRILL-858: Fix bugs in handling Hive views in INFORMATION_SCHEMA generation.
Also:
- Added a new interface DrillViewInfoProvider which provides view related info to
components such as InfoSchema. Drill or Hive view table implement this interface.
- Throw UnsupportedOperationException if any queries use Hive views
as querying Hive views is not supported in current version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60a429fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60a429fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60a429fb
Branch: refs/heads/master
Commit: 60a429fb7815c655234055ded237dbda1af2d769
Parents: b328d7b
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 17:13:59 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:12:44 2014 -0700
----------------------------------------------------------------------
.../planner/logical/DrillViewInfoProvider.java | 28 ++++++++++++++
.../exec/planner/logical/DrillViewTable.java | 3 +-
.../drill/exec/store/hive/HiveReadEntry.java | 9 +++++
.../exec/store/hive/HiveStoragePlugin.java | 5 +++
.../exec/store/hive/schema/DrillHiveTable.java | 2 +-
.../store/hive/schema/DrillHiveViewTable.java | 40 ++++++++++++++++++++
.../store/hive/schema/HiveSchemaFactory.java | 7 +++-
.../exec/store/ischema/RecordGenerator.java | 4 +-
.../exec/store/hive/HiveTestDataGenerator.java | 3 ++
.../apache/drill/jdbc/test/TestMetadataDDL.java | 2 +
.../org/apache/drill/jdbc/test/TestViews.java | 8 ++++
11 files changed, 106 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java
new file mode 100644
index 0000000..50e1d8f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+/**
+ * Interface used by Drill components such as InformationSchema generator to get view info.
+ * All view tables need to implement this interface.
+ */
+public interface DrillViewInfoProvider {
+
+ /** Get the query part of the view. */
+ String getViewSql();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
index aaf32ad..19fd7ba 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
@@ -32,7 +32,7 @@ import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeFactory;
-public class DrillViewTable implements TranslatableTable{
+public class DrillViewTable implements TranslatableTable, DrillViewInfoProvider {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillViewTable.class);
private final View view;
@@ -71,6 +71,7 @@ public class DrillViewTable implements TranslatableTable{
return TableType.VIEW;
}
+ @Override
public String getViewSql() {
return view.getSql();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
index f330a1e..855b05f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.hive;
import java.util.List;
+import net.hydromatic.optiq.Schema.TableType;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.Size;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -60,5 +61,13 @@ public class HiveReadEntry {
return partitionsUnwrapped;
}
+ @JsonIgnore
+ public TableType getJdbcTableType() {
+ if (table.getTable().getTableType().equals(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW.toString())) {
+ return TableType.VIEW;
+ }
+
+ return TableType.TABLE;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
index 6d5f6d2..c5a6e2c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Schema.TableType;
import net.hydromatic.optiq.SchemaPlus;
import org.apache.drill.common.JSONOptions;
@@ -67,6 +68,10 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {
public HiveScan getPhysicalScan(JSONOptions selection, List<SchemaPath> columns) throws IOException {
HiveReadEntry hiveReadEntry = selection.getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
try {
+ if (hiveReadEntry.getJdbcTableType() == TableType.VIEW) {
+ throw new UnsupportedOperationException("Querying Hive views from Drill is not supported in current version.");
+ }
+
return new HiveScan(hiveReadEntry, this, null);
} catch (ExecutionSetupException e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index d1a5659..547c0bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -41,7 +41,7 @@ import org.eigenbase.sql.type.SqlTypeName;
public class DrillHiveTable extends DrillTable{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveTable.class);
- private final Table hiveTable;
+ protected final Table hiveTable;
public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) {
super(storageEngineName, plugin, readEntry);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java
new file mode 100644
index 0000000..b575972
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive.schema;
+
+import net.hydromatic.optiq.Schema.TableType;
+import org.apache.drill.exec.planner.logical.DrillViewInfoProvider;
+import org.apache.drill.exec.store.hive.HiveReadEntry;
+import org.apache.drill.exec.store.hive.HiveStoragePlugin;
+
+public class DrillHiveViewTable extends DrillHiveTable implements DrillViewInfoProvider {
+
+ public DrillHiveViewTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) {
+ super(storageEngineName, plugin, readEntry);
+ }
+
+ @Override
+ public TableType getJdbcTableType() {
+ return TableType.VIEW;
+ }
+
+ @Override
+ public String getViewSql() {
+ return hiveTable.getViewExpandedText();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
index fb672d4..7e6b92b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -258,7 +258,12 @@ public class HiveSchemaFactory implements SchemaFactory {
DrillTable getDrillTable(String dbName, String t){
HiveReadEntry entry = getSelectionBaseOnName(dbName, t);
if(entry == null) return null;
- return new DrillHiveTable(schemaName, plugin, entry);
+
+ if (entry.getJdbcTableType() == TableType.VIEW) {
+ return new DrillHiveViewTable(schemaName, plugin, entry);
+ } else {
+ return new DrillHiveTable(schemaName, plugin, entry);
+ }
}
HiveReadEntry getSelectionBaseOnName(String dbName, String t) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
index a8792fe..df67da1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
@@ -23,7 +23,7 @@ import net.hydromatic.optiq.Schema.TableType;
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.Table;
import net.hydromatic.optiq.jdbc.JavaTypeFactoryImpl;
-import org.apache.drill.exec.planner.logical.DrillViewTable;
+import org.apache.drill.exec.planner.logical.DrillViewInfoProvider;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.pojo.PojoRecordReader;
@@ -148,7 +148,7 @@ public abstract class RecordGenerator {
@Override
public boolean visitTable(String schemaName, String tableName, Table table) {
if (table.getJdbcTableType() == TableType.VIEW) {
- records.add(new Records.View("DRILL", schemaName, tableName, ((DrillViewTable)table).getViewSql()));
+ records.add(new Records.View("DRILL", schemaName, tableName, ((DrillViewInfoProvider) table).getViewSql()));
}
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 8433931..f914661 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -118,6 +118,9 @@ public class HiveTestDataGenerator {
"uniontypeType UNIONTYPE<int, double, array<string>>)"
);
+ // create a Hive view to test how its metadata is populated in Drill's INFORMATION_SCHEMA
+ executeQuery("CREATE VIEW IF NOT EXISTS hiveview AS SELECT * FROM kv");
+
ss.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 501556a..123c160 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -51,6 +51,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"
@@ -74,6 +75,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 7dc9c30..93626d5 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -302,6 +302,14 @@ public class TestViews extends TestJdbcQuery {
}
@Test
+ public void testInfoSchemaWithHiveView() throws Exception {
+ JdbcAssert.withFull("hive.default")
+ .sql("SELECT * FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_NAME = 'hiveview'")
+ .returns("TABLE_CATALOG=DRILL; TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview; " +
+ "VIEW_DEFINITION=SELECT `kv`.`key`, `kv`.`value` FROM `default`.`kv`");
+ }
+
+ @Test
public void testViewWithFullSchemaIdentifier() throws Exception{
JdbcAssert.withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
public Void apply(Connection connection) {