You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2018/06/04 22:38:33 UTC
hive git commit: HIVE-19467: Make storage format configurable for
temp tables created using LLAP external client (Jason Dere,
reviewed by Deepak Jaiswal)
Repository: hive
Updated Branches:
refs/heads/branch-3 5ec8e356d -> ac5de3d45
HIVE-19467: Make storage format configurable for temp tables created using LLAP external client (Jason Dere, reviewed by Deepak Jaiswal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ac5de3d4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ac5de3d4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ac5de3d4
Branch: refs/heads/branch-3
Commit: ac5de3d45ce33ed61a00f5a2f3637cf755d3ad82
Parents: 5ec8e35
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed May 9 18:05:50 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Jun 4 15:30:56 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java | 13 +++++++++++++
.../hive/ql/udf/generic/GenericUDTFGetSplits.java | 15 ++++++++++++++-
3 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ac5de3d4/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 8347f7f..6939dd0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4149,6 +4149,9 @@ public class HiveConf extends Configuration {
LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES("hive.llap.daemon.output.service.max.pending.writes",
8, "Maximum number of queued writes allowed per connection when sending data\n" +
" via the LLAP output service to external clients."),
+ LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format",
+ "orc", new StringSet("default", "text", "orc"),
+ "Storage format for temp tables created using LLAP external client"),
LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false,
"Override if grace join should be allowed to run in llap."),
http://git-wip-us.apache.org/repos/asf/hive/blob/ac5de3d4/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
index 11017f6..7a891ef 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
@@ -448,6 +448,19 @@ public abstract class BaseJdbcWithMiniLlap {
assertArrayEquals("X'01FF'".getBytes("UTF-8"), (byte[]) rowValues[22]);
}
+
+ @Test(timeout = 60000)
+ public void testComplexQuery() throws Exception {
+ createTestTable("testtab1");
+
+ RowCollector rowCollector = new RowCollector();
+ String query = "select value, count(*) from testtab1 where under_col=0 group by value";
+ int rowCount = processQuery(query, 1, rowCollector);
+ assertEquals(1, rowCount);
+
+ assertArrayEquals(new String[] {"val_0", "3"}, rowCollector.rows.get(0));
+ }
+
private interface RowProcessor {
void process(Row row);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ac5de3d4/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index 7dbde7a..20d0961 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -293,7 +293,8 @@ public class GenericUDTFGetSplits extends GenericUDTF {
String tableName = "table_"+UUID.randomUUID().toString().replaceAll("[^A-Za-z0-9 ]", "");
- String ctas = "create temporary table " + tableName + " as " + query;
+ String storageFormatString = getTempTableStorageFormatString(conf);
+ String ctas = "create temporary table " + tableName + " " + storageFormatString + " as " + query;
LOG.info("Materializing the query for LLAPIF; CTAS: " + ctas);
driver.releaseResources();
HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, originalMode);
@@ -674,6 +675,18 @@ public class GenericUDTFGetSplits extends GenericUDTF {
return Schema;
}
+ private String getTempTableStorageFormatString(HiveConf conf) {
+ String formatString = "";
+ String storageFormatOption =
+ conf.getVar(HiveConf.ConfVars.LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT).toLowerCase();
+ if (storageFormatOption.equals("text")) {
+ formatString = "stored as textfile";
+ } else if (storageFormatOption.equals("orc")) {
+ formatString = "stored as orc";
+ }
+ return formatString;
+ }
+
@Override
public void close() throws HiveException {
}