You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2018/05/10 01:06:20 UTC
hive git commit: HIVE-19467: Make storage format configurable for
temp tables created using LLAP external client (Jason Dere,
reviewed by Deepak Jaiswal)
Repository: hive
Updated Branches:
refs/heads/master 8ac625744 -> 1cd5274c5
HIVE-19467: Make storage format configurable for temp tables created using LLAP external client (Jason Dere, reviewed by Deepak Jaiswal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1cd5274c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1cd5274c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1cd5274c
Branch: refs/heads/master
Commit: 1cd5274c58eb53cae8f38d172bbbc1517f199251
Parents: 8ac6257
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed May 9 18:05:50 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Wed May 9 18:05:50 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../org/apache/hive/jdbc/TestJdbcWithMiniLlap.java | 13 +++++++++++++
.../hive/ql/udf/generic/GenericUDTFGetSplits.java | 15 ++++++++++++++-
3 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd5274c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index fa3f788..cc490af 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4145,6 +4145,9 @@ public class HiveConf extends Configuration {
LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES("hive.llap.daemon.output.service.max.pending.writes",
8, "Maximum number of queued writes allowed per connection when sending data\n" +
" via the LLAP output service to external clients."),
+ LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format",
+ "orc", new StringSet("default", "text", "orc"),
+ "Storage format for temp tables created using LLAP external client"),
LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false,
"Override if grace join should be allowed to run in llap."),
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd5274c/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
index 68a8e21..7e35fef 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
@@ -439,6 +439,19 @@ public class TestJdbcWithMiniLlap {
assertArrayEquals("X'01FF'".getBytes("UTF-8"), (byte[]) rowValues[22]);
}
+
+ @Test(timeout = 60000)
+ public void testComplexQuery() throws Exception {
+ createTestTable("testtab1");
+
+ RowCollector rowCollector = new RowCollector();
+ String query = "select value, count(*) from testtab1 where under_col=0 group by value";
+ int rowCount = processQuery(query, 1, rowCollector);
+ assertEquals(1, rowCount);
+
+ assertArrayEquals(new String[] {"val_0", "3"}, rowCollector.rows.get(0));
+ }
+
private interface RowProcessor {
void process(Row row);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd5274c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index cae02a9..e74a188 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -268,7 +268,8 @@ public class GenericUDTFGetSplits extends GenericUDTF {
String tableName = "table_"+UUID.randomUUID().toString().replaceAll("[^A-Za-z0-9 ]", "");
- String ctas = "create temporary table " + tableName + " as " + query;
+ String storageFormatString = getTempTableStorageFormatString(conf);
+ String ctas = "create temporary table " + tableName + " " + storageFormatString + " as " + query;
LOG.info("Materializing the query for LLAPIF; CTAS: " + ctas);
driver.releaseResources();
HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, originalMode);
@@ -641,6 +642,18 @@ public class GenericUDTFGetSplits extends GenericUDTF {
return Schema;
}
+ private String getTempTableStorageFormatString(HiveConf conf) {
+ String formatString = "";
+ String storageFormatOption =
+ conf.getVar(HiveConf.ConfVars.LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT).toLowerCase();
+ if (storageFormatOption.equals("text")) {
+ formatString = "stored as textfile";
+ } else if (storageFormatOption.equals("orc")) {
+ formatString = "stored as orc";
+ }
+ return formatString;
+ }
+
@Override
public void close() throws HiveException {
}