You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ve...@apache.org on 2015/06/17 23:31:06 UTC
[3/3] drill git commit: DRILL-3266: Add hive-contrib dependency to
Hive storage plugin
DRILL-3266: Add hive-contrib dependency to Hive storage plugin
Module hive-contrib has SerDes and UDF implementations which we want them to
be available by default in Drill Hive storage plugin.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d32acdb9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d32acdb9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d32acdb9
Branch: refs/heads/master
Commit: d32acdb9020aaaba656588a03a02dd42e6a61241
Parents: b46c53e
Author: vkorukanti <ve...@gmail.com>
Authored: Mon Jun 15 08:11:10 2015 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Wed Jun 17 14:25:35 2015 -0700
----------------------------------------------------------------------
contrib/storage-hive/core/pom.xml | 4 +++
.../apache/drill/exec/hive/TestHiveStorage.java | 10 ++++++++
.../exec/hive/TestInfoSchemaOnHiveStorage.java | 20 ++++++++++++---
.../exec/store/hive/HiveTestDataGenerator.java | 24 +++++++++--------
pom.xml | 27 ++++++++++++++++++++
5 files changed, 71 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index fb96a0d..546fd7b 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -61,6 +61,10 @@
<artifactId>hive-hbase-handler</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-contrib</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 2898f91..39e8f54 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -186,4 +186,14 @@ public class TestHiveStorage extends HiveTestBase {
.baselineValues(5, " key_5")
.go();
}
+
+ @Test // DRILL-3266
+ public void queryingTableWithSerDeInHiveContribJar() throws Exception {
+ testBuilder()
+ .sqlQuery("SELECT * FROM hive.db1.kv_db1 ORDER BY key DESC LIMIT 1")
+ .unOrdered()
+ .baselineColumns("key", "value")
+ .baselineValues("5", " key_5")
+ .go();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index b684538..6118be5 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -93,7 +93,13 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
// When table name is fully qualified with schema name (sub-schema is non-default schema)
@Test
public void describeTable2() throws Exception{
- describeHelper(null, "DESCRIBE hive.`db1`.kv_db1");
+ testBuilder()
+ .sqlQuery("DESCRIBE hive.`db1`.kv_db1")
+ .unOrdered()
+ .baselineColumns(baselineCols)
+ .baselineValues("key", "CHARACTER VARYING", "YES")
+ .baselineValues("value", "CHARACTER VARYING", "YES")
+ .go();
}
// When table is qualified with just the top level schema. It should look for the table in default sub-schema within
@@ -113,7 +119,13 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
// given as single level schema name.
@Test
public void describeTable5() throws Exception {
- describeHelper(null, "DESCRIBE `hive.db1`.kv_db1");
+ testBuilder()
+ .sqlQuery("DESCRIBE `hive.db1`.kv_db1")
+ .unOrdered()
+ .baselineColumns(baselineCols)
+ .baselineValues("key", "CHARACTER VARYING", "YES")
+ .baselineValues("value", "CHARACTER VARYING", "YES")
+ .go();
}
// When current default schema is just the top-level schema name and the table has no schema qualifier. It should
@@ -187,8 +199,8 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
.unOrdered()
.optionSettingQueriesForTestQuery("USE hive.db1")
.baselineColumns("key", "value")
- .baselineValues(1, " key_1")
- .baselineValues(2, " key_2")
+ .baselineValues("1", " key_1")
+ .baselineValues("2", " key_2")
.go();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 5b60c05..ea8d90f 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -130,9 +130,21 @@ public class HiveTestDataGenerator {
// generate (key, value) test data
String testDataFile = generateTestDataFile();
- createTableAndLoadData(hiveDriver, "default", "kv", testDataFile);
+ // Create a (key, value) schema table with Text SerDe which is available in hive-serdes.jar
+ executeQuery(hiveDriver, "CREATE TABLE IF NOT EXISTS default.kv(key INT, value STRING) " +
+ "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE");
+ executeQuery(hiveDriver, "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE default.kv");
+
+ // Create a (key, value) schema table in non-default database with RegexSerDe which is available in hive-contrib.jar
+ // Table with RegExSerde is expected to have columns of STRING type only.
executeQuery(hiveDriver, "CREATE DATABASE IF NOT EXISTS db1");
- createTableAndLoadData(hiveDriver, "db1", "kv_db1", testDataFile);
+ executeQuery(hiveDriver, "CREATE TABLE db1.kv_db1(key STRING, value STRING) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' " +
+ "WITH SERDEPROPERTIES (" +
+ " \"input.regex\" = \"([0-9]*), (.*_[0-9]*)\", " +
+ " \"output.format.string\" = \"%1$s, %2$s\"" +
+ ") ");
+ executeQuery(hiveDriver, "INSERT INTO TABLE db1.kv_db1 SELECT * FROM default.kv");
// Create an Avro format based table backed by schema in a separate file
final String avroCreateQuery = String.format("CREATE TABLE db1.avro " +
@@ -293,14 +305,6 @@ public class HiveTestDataGenerator {
ss.close();
}
- private void createTableAndLoadData(Driver hiveDriver, String dbName, String tblName, String dataFile) {
- executeQuery(hiveDriver, String.format("USE %s", dbName));
- executeQuery(hiveDriver, String.format("CREATE TABLE IF NOT EXISTS %s.%s(key INT, value STRING) "+
- "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", dbName, tblName));
- executeQuery(hiveDriver,
- String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE %s.%s", dataFile, dbName, tblName));
- }
-
private File getTempFile() throws Exception {
return java.nio.file.Files.createTempFile("drill-hive-test", ".txt").toFile();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1b14ae1..675a6e0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -711,6 +711,33 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-contrib</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-shims</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</dependencyManagement>