You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/10/08 21:35:56 UTC
[1/9] hive git commit: HIVE-12011: unable to create temporary table
using CTAS if regular table with that name already exists (Pengcheng Xiong,
reviewed by Laljo John Pullokkaran)
Repository: hive
Updated Branches:
refs/heads/llap 7a2391376 -> be00a031d
HIVE-12011: unable to create temporary table using CTAS if regular table with that name already exists (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b2f63ba9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b2f63ba9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b2f63ba9
Branch: refs/heads/llap
Commit: b2f63ba914045e8285445d48bf656627a84bd2f7
Parents: bcff871
Author: Pengcheng Xiong <px...@apache.org>
Authored: Tue Oct 6 14:19:12 2015 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Tue Oct 6 14:19:12 2015 -0700
----------------------------------------------------------------------
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 31 ++++--
ql/src/test/queries/clientpositive/temp_table.q | 26 +++++
.../clientpositive/spark/temp_table.q.out | 107 +++++++++++++++++++
.../results/clientpositive/temp_table.q.out | 107 +++++++++++++++++++
.../results/clientpositive/tez/temp_table.q.out | 107 +++++++++++++++++++
6 files changed, 372 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b2f63ba9/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 51ff262..6091c3f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -515,7 +515,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
return newCopy;
}
- private Map<String, Table> getTempTablesForDatabase(String dbName) {
+ public static Map<String, Table> getTempTablesForDatabase(String dbName) {
SessionState ss = SessionState.get();
if (ss == null) {
LOG.debug("No current SessionState, skipping temp tables");
http://git-wip-us.apache.org/repos/asf/hive/blob/b2f63ba9/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 4bec228..7a54aec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -112,6 +112,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.Optimizer;
@@ -10943,14 +10944,30 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
case CTAS: // create table as select
- // Verify that the table does not already exist
- try {
- Table dumpTable = db.newTable(dbDotTab);
- if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false)) {
- throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTab));
+ if (isTemporary) {
+ String dbName = qualifiedTabName[0];
+ String tblName = qualifiedTabName[1];
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ throw new SemanticException("No current SessionState, cannot create temporary table "
+ + dbName + "." + tblName);
+ }
+ Map<String, Table> tables = SessionHiveMetaStoreClient.getTempTablesForDatabase(dbName);
+ if (tables != null && tables.containsKey(tblName)) {
+ throw new SemanticException("Temporary table " + dbName + "." + tblName
+ + " already exists");
+ }
+ } else {
+ // Verify that the table does not already exist
+ // dumpTable is only used to check the conflict for non-temporary tables
+ try {
+ Table dumpTable = db.newTable(dbDotTab);
+ if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false)) {
+ throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTab));
+ }
+ } catch (HiveException e) {
+ throw new SemanticException(e);
}
- } catch (HiveException e) {
- throw new SemanticException(e);
}
if(location != null && location.length() != 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b2f63ba9/ql/src/test/queries/clientpositive/temp_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/temp_table.q b/ql/src/test/queries/clientpositive/temp_table.q
index e587f3f..65f3eb4 100644
--- a/ql/src/test/queries/clientpositive/temp_table.q
+++ b/ql/src/test/queries/clientpositive/temp_table.q
@@ -42,3 +42,29 @@ use default;
DROP DATABASE two CASCADE;
DROP TABLE bay;
+
+create table s as select * from src limit 10;
+
+select count(*) from s;
+
+create temporary table s as select * from s limit 2;
+
+select count(*) from s;
+
+with s as ( select * from src limit 1)
+select count(*) from s;
+
+with src as ( select * from s)
+select count(*) from src;
+
+drop table s;
+
+select count(*) from s;
+
+with s as ( select * from src limit 1)
+select count(*) from s;
+
+with src as ( select * from s)
+select count(*) from src;
+
+drop table s;
http://git-wip-us.apache.org/repos/asf/hive/blob/b2f63ba9/ql/src/test/results/clientpositive/spark/temp_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/temp_table.q.out b/ql/src/test/results/clientpositive/spark/temp_table.q.out
index 65e256d..718a8a4 100644
--- a/ql/src/test/results/clientpositive/spark/temp_table.q.out
+++ b/ql/src/test/results/clientpositive/spark/temp_table.q.out
@@ -448,3 +448,110 @@ POSTHOOK: query: DROP TABLE bay
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@bay
POSTHOOK: Output: default@bay
+PREHOOK: query: create table s as select * from src limit 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@s
+POSTHOOK: query: create table s as select * from src limit 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: create temporary table s as select * from s limit 2
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@s
+PREHOOK: Output: database:default
+PREHOOK: Output: default@s
+POSTHOOK: query: create temporary table s as select * from s limit 2
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@s
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+2
+PREHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+PREHOOK: query: with src as ( select * from s)
+select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: with src as ( select * from s)
+select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+2
+PREHOOK: query: drop table s
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@s
+PREHOOK: Output: default@s
+POSTHOOK: query: drop table s
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@s
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+PREHOOK: query: with src as ( select * from s)
+select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: with src as ( select * from s)
+select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: drop table s
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@s
+PREHOOK: Output: default@s
+POSTHOOK: query: drop table s
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@s
+POSTHOOK: Output: default@s
http://git-wip-us.apache.org/repos/asf/hive/blob/b2f63ba9/ql/src/test/results/clientpositive/temp_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/temp_table.q.out b/ql/src/test/results/clientpositive/temp_table.q.out
index e2987fe..a9f2bae 100644
--- a/ql/src/test/results/clientpositive/temp_table.q.out
+++ b/ql/src/test/results/clientpositive/temp_table.q.out
@@ -520,3 +520,110 @@ POSTHOOK: query: DROP TABLE bay
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@bay
POSTHOOK: Output: default@bay
+PREHOOK: query: create table s as select * from src limit 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@s
+POSTHOOK: query: create table s as select * from src limit 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: create temporary table s as select * from s limit 2
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@s
+PREHOOK: Output: database:default
+PREHOOK: Output: default@s
+POSTHOOK: query: create temporary table s as select * from s limit 2
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@s
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+2
+PREHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+PREHOOK: query: with src as ( select * from s)
+select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: with src as ( select * from s)
+select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+2
+PREHOOK: query: drop table s
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@s
+PREHOOK: Output: default@s
+POSTHOOK: query: drop table s
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@s
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+PREHOOK: query: with src as ( select * from s)
+select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: with src as ( select * from s)
+select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: drop table s
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@s
+PREHOOK: Output: default@s
+POSTHOOK: query: drop table s
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@s
+POSTHOOK: Output: default@s
http://git-wip-us.apache.org/repos/asf/hive/blob/b2f63ba9/ql/src/test/results/clientpositive/tez/temp_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/temp_table.q.out b/ql/src/test/results/clientpositive/tez/temp_table.q.out
index 49f57c2..200ccdd 100644
--- a/ql/src/test/results/clientpositive/tez/temp_table.q.out
+++ b/ql/src/test/results/clientpositive/tez/temp_table.q.out
@@ -460,3 +460,110 @@ POSTHOOK: query: DROP TABLE bay
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@bay
POSTHOOK: Output: default@bay
+PREHOOK: query: create table s as select * from src limit 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@s
+POSTHOOK: query: create table s as select * from src limit 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: create temporary table s as select * from s limit 2
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@s
+PREHOOK: Output: database:default
+PREHOOK: Output: default@s
+POSTHOOK: query: create temporary table s as select * from s limit 2
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@s
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+2
+PREHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+PREHOOK: query: with src as ( select * from s)
+select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: with src as ( select * from s)
+select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+2
+PREHOOK: query: drop table s
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@s
+PREHOOK: Output: default@s
+POSTHOOK: query: drop table s
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@s
+POSTHOOK: Output: default@s
+PREHOOK: query: select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with s as ( select * from src limit 1)
+select count(*) from s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+PREHOOK: query: with src as ( select * from s)
+select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@s
+#### A masked pattern was here ####
+POSTHOOK: query: with src as ( select * from s)
+select count(*) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@s
+#### A masked pattern was here ####
+10
+PREHOOK: query: drop table s
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@s
+PREHOOK: Output: default@s
+POSTHOOK: query: drop table s
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@s
+POSTHOOK: Output: default@s
[2/9] hive git commit: HIVE-11977 : Hive should handle an external
avro table with zero length files present (Aaron Dossett via Ashutosh
Chauhan)
Posted by se...@apache.org.
HIVE-11977 : Hive should handle an external avro table with zero length files present (Aaron Dossett via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1e0c508d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1e0c508d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1e0c508d
Branch: refs/heads/llap
Commit: 1e0c508d7335b255a7f2638730e2fff8494809f9
Parents: b2f63ba
Author: Aaron Dossett <do...@apache.org>
Authored: Mon Sep 28 11:30:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Oct 7 09:25:14 2015 -0700
----------------------------------------------------------------------
.../ql/io/avro/AvroGenericRecordReader.java | 22 ++++++--
.../ql/io/avro/TestAvroGenericRecordReader.java | 59 ++++++++++++++++++++
2 files changed, 75 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1e0c508d/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
index 1381514..8d58d74 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
@@ -57,6 +57,7 @@ public class AvroGenericRecordReader implements
final private long start;
final private long stop;
protected JobConf jobConf;
+ final private boolean isEmptyInput;
/**
* A unique ID for each record reader.
*/
@@ -78,9 +79,17 @@ public class AvroGenericRecordReader implements
gdr.setExpected(latest);
}
- this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
- this.reader.sync(split.getStart());
- this.start = reader.tell();
+ if (split.getLength() == 0) {
+ this.isEmptyInput = true;
+ this.start = 0;
+ this.reader = null;
+ }
+ else {
+ this.isEmptyInput = false;
+ this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
+ this.reader.sync(split.getStart());
+ this.start = reader.tell();
+ }
this.stop = split.getStart() + split.getLength();
this.recordReaderID = new UID();
}
@@ -146,7 +155,7 @@ public class AvroGenericRecordReader implements
@Override
public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException {
- if(!reader.hasNext() || reader.pastSync(stop)) {
+ if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) {
return false;
}
@@ -170,12 +179,13 @@ public class AvroGenericRecordReader implements
@Override
public long getPos() throws IOException {
- return reader.tell();
+ return isEmptyInput ? 0 : reader.tell();
}
@Override
public void close() throws IOException {
- reader.close();
+ if (isEmptyInput == false)
+ reader.close();
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/1e0c508d/ql/src/test/org/apache/hadoop/hive/ql/io/avro/TestAvroGenericRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/avro/TestAvroGenericRecordReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/avro/TestAvroGenericRecordReader.java
new file mode 100644
index 0000000..6d4356a
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/avro/TestAvroGenericRecordReader.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.avro;
+
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+public class TestAvroGenericRecordReader {
+
+ @Mock private JobConf jobConf;
+ @Mock private FileSplit emptyFileSplit;
+ @Mock private Reporter reporter;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ when(emptyFileSplit.getLength()).thenReturn(0l);
+ }
+
+ @Test
+ public void emptyFile() throws IOException
+ {
+ AvroGenericRecordReader reader = new AvroGenericRecordReader(jobConf, emptyFileSplit, reporter);
+
+ //next() should always return false
+ Assert.assertEquals(false, reader.next(null, null));
+
+ //getPos() should always return 0
+ Assert.assertEquals(0, reader.getPos());
+
+ //close() should just do nothing
+ reader.close();
+ }
+}
[3/9] hive git commit: HIVE-12018 : beeline --help doesn't return to
original prompt (Mohammad Islam via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-12018 : beeline --help doesn't return to original prompt (Mohammad Islam via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/556877c2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/556877c2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/556877c2
Branch: refs/heads/llap
Commit: 556877c24a335f207d74bb7daa6949af227ffe60
Parents: 1e0c508
Author: Mohammad Kamrul Islam <mi...@yahoo.com>
Authored: Fri Oct 2 15:00:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Oct 7 09:31:33 2015 -0700
----------------------------------------------------------------------
beeline/src/java/org/apache/hive/beeline/BeeLine.java | 4 ++++
beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java | 9 +++++++++
.../test/org/apache/hive/beeline/TestBeelineArgParsing.java | 1 +
3 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/556877c2/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index f2aeac5..69e9418 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -727,6 +727,7 @@ public class BeeLine implements Closeable {
if (cl.hasOption("help")) {
usage();
+ getOpts().setHelpAsked(true);
return 0;
}
@@ -844,6 +845,9 @@ public class BeeLine implements Closeable {
defaultConnect(false);
}
+ if (getOpts().isHelpAsked()) {
+ return 0;
+ }
if (getOpts().getScriptFile() != null) {
return executeFile(getOpts().getScriptFile());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/556877c2/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
index 8e1d11b..7a6ee5f 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
@@ -100,6 +100,7 @@ class BeeLineOpts implements Completer {
private Map<String, String> hiveVariables = new HashMap<String, String>();
private Map<String, String> hiveConfVariables = new HashMap<String, String>();
+ private boolean helpAsked;
public BeeLineOpts(BeeLine beeLine, Properties props) {
this.beeLine = beeLine;
@@ -558,5 +559,13 @@ class BeeLineOpts implements Completer {
public HiveConf getConf() {
return conf;
}
+
+ public void setHelpAsked(boolean helpAsked) {
+ this.helpAsked = helpAsked;
+ }
+
+ public boolean isHelpAsked() {
+ return helpAsked;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/556877c2/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
index 702805f..e529057 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
@@ -206,6 +206,7 @@ public class TestBeelineArgParsing {
TestBeeline bl = new TestBeeline();
String args[] = new String[] {"--help"};
Assert.assertEquals(0, bl.initArgs(args));
+ Assert.assertEquals(true, bl.getOpts().isHelpAsked());
}
/**
[8/9] hive git commit: HIVE-11149 : Fix issue with sometimes HashMap
in PerfLogger.java hangs (WangMeng, reviewed by Xuefu Zhang, Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11149 : Fix issue with sometimes HashMap in PerfLogger.java hangs (WangMeng, reviewed by Xuefu Zhang, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aded0d32
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aded0d32
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aded0d32
Branch: refs/heads/llap
Commit: aded0d32dcffd7b333fe20211045fbb221a28279
Parents: 48c420e
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 8 11:13:20 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 8 11:35:43 2015 -0700
----------------------------------------------------------------------
.../src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java | 2 +-
.../java/org/apache/hadoop/hive/ql/session/SessionState.java | 8 +-------
2 files changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/aded0d32/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
index 6263a6d..67b2282 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -84,7 +84,7 @@ public class PerfLogger {
protected static final ThreadLocal<PerfLogger> perfLogger = new ThreadLocal<PerfLogger>();
- public PerfLogger() {
+ private PerfLogger() {
// Use getPerfLogger to get an instance of PerfLogger
}
http://git-wip-us.apache.org/repos/asf/hive/blob/aded0d32/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 56b0fae..41b4bb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -219,8 +219,6 @@ public class SessionState {
*/
LineageState ls;
- private PerfLogger perfLogger;
-
private final String userName;
/**
@@ -1589,12 +1587,8 @@ public class SessionState {
SessionState ss = get();
if (ss == null) {
return PerfLogger.getPerfLogger(null, resetPerfLogger);
- } else if (ss.perfLogger != null && !resetPerfLogger) {
- return ss.perfLogger;
} else {
- PerfLogger perfLogger = PerfLogger.getPerfLogger(ss.getConf(), resetPerfLogger);
- ss.perfLogger = perfLogger;
- return perfLogger;
+ return PerfLogger.getPerfLogger(ss.getConf(), resetPerfLogger);
}
}
[6/9] hive git commit: HIVE-9695: Redundant filter operator in
reducer Vertex when CBO is disabled (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
index 7d9d99e..141a80b 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
@@ -548,10 +548,10 @@ Stage-0
Select Operator [SEL_37]
outputColumnNames:["_col2","_col6"]
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_36]
+ Filter Operator [FIL_51]
predicate:((_col1 > 0) or (_col6 >= 0)) (type: boolean)
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_55]
+ Merge Join Operator [MERGEJOIN_57]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col6"]
@@ -566,7 +566,7 @@ Stage-0
| Select Operator [SEL_30]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_53]
+ | Filter Operator [FIL_55]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_29]
@@ -582,10 +582,10 @@ Stage-0
Select Operator [SEL_28]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_27]
+ Filter Operator [FIL_52]
predicate:((_col1 + _col4) >= 0) (type: boolean)
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_54]
+ Merge Join Operator [MERGEJOIN_56]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2","_col4"]
@@ -626,7 +626,7 @@ Stage-0
| keys:key (type: string), c_int (type: int), c_float (type: float)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_52]
+ | Filter Operator [FIL_54]
| predicate:((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and key is not null) (type: boolean)
| Statistics:Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_11]
@@ -668,7 +668,7 @@ Stage-0
keys:key (type: string), c_int (type: int), c_float (type: float)
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_51]
+ Filter Operator [FIL_53]
predicate:((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and key is not null) (type: boolean)
Statistics:Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_0]
@@ -731,10 +731,10 @@ Stage-0
Select Operator [SEL_34]
outputColumnNames:["_col2","_col6"]
Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_48]
+ Filter Operator [FIL_49]
predicate:((((_col6 > 0) and ((_col6 >= 1) or (_col2 >= 1))) and ((UDFToLong(_col6) + _col2) >= 0)) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_53]
+ Merge Join Operator [MERGEJOIN_55]
| condition map:[{"":"Left Outer Join0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col6"]
@@ -762,10 +762,10 @@ Stage-0
Select Operator [SEL_27]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_26]
+ Filter Operator [FIL_50]
predicate:((_col1 + _col4) >= 0) (type: boolean)
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_52]
+ Merge Join Operator [MERGEJOIN_54]
| condition map:[{"":"Left Outer Join0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2","_col4"]
@@ -809,7 +809,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_49]
+ | Filter Operator [FIL_51]
| predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -842,7 +842,7 @@ Stage-0
Select Operator [SEL_15]
outputColumnNames:["_col0","_col2","_col3"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_50]
+ Filter Operator [FIL_52]
predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_13]
@@ -894,10 +894,10 @@ Stage-0
Select Operator [SEL_30]
outputColumnNames:["_col2","_col6"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_29]
+ Filter Operator [FIL_38]
predicate:(((_col1 + _col4) >= 2) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_40]
+ Merge Join Operator [MERGEJOIN_41]
| condition map:[{"":"Right Outer Join0 to 1"},{"":"Right Outer Join0 to 2"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)","2":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col4","_col6"]
@@ -954,7 +954,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_38]
+ | Filter Operator [FIL_39]
| predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -987,7 +987,7 @@ Stage-0
Select Operator [SEL_15]
outputColumnNames:["_col0","_col2","_col3"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_39]
+ Filter Operator [FIL_40]
predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_13]
@@ -1050,10 +1050,10 @@ Stage-0
Select Operator [SEL_33]
outputColumnNames:["_col2","_col6"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_44]
+ Filter Operator [FIL_45]
predicate:(((((_col6 > 0) and ((_col6 >= 1) or (_col2 >= 1))) and ((UDFToLong(_col6) + _col2) >= 0)) and ((_col1 + _col4) >= 0)) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_47]
+ Merge Join Operator [MERGEJOIN_48]
| condition map:[{"":"Outer Join 0 to 1"},{"":"Outer Join 0 to 2"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)","2":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col4","_col6"]
@@ -1110,7 +1110,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_45]
+ | Filter Operator [FIL_46]
| predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -1155,7 +1155,7 @@ Stage-0
Select Operator [SEL_15]
outputColumnNames:["_col0","_col2","_col3"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_46]
+ Filter Operator [FIL_47]
predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_13]
@@ -1207,10 +1207,10 @@ Stage-0
Select Operator [SEL_35]
outputColumnNames:["_col2","_col6"]
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_34]
+ Filter Operator [FIL_46]
predicate:((_col1 > 0) or (_col6 >= 0)) (type: boolean)
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_50]
+ Merge Join Operator [MERGEJOIN_52]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col6"]
@@ -1225,7 +1225,7 @@ Stage-0
| Select Operator [SEL_28]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_48]
+ | Filter Operator [FIL_50]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_27]
@@ -1241,10 +1241,10 @@ Stage-0
Select Operator [SEL_26]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_25]
+ Filter Operator [FIL_47]
predicate:((_col1 + _col4) >= 0) (type: boolean)
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_49]
+ Merge Join Operator [MERGEJOIN_51]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2","_col4"]
@@ -1279,7 +1279,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_46]
+ | Filter Operator [FIL_48]
| predicate:(((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) and key is not null) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -1312,7 +1312,7 @@ Stage-0
Select Operator [SEL_12]
outputColumnNames:["_col0","_col2","_col3"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_47]
+ Filter Operator [FIL_49]
predicate:(((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) and key is not null) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_10]
@@ -1954,10 +1954,10 @@ Stage-0
Select Operator [SEL_19]
outputColumnNames:["_col0","_col1","_col2","_col3","_col4"]
Statistics:Num rows: 4 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_18]
+ Filter Operator [FIL_26]
predicate:(((_col4 + 1) = 2) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 4 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_32]
+ Merge Join Operator [MERGEJOIN_34]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4","_col6"]
@@ -1972,7 +1972,7 @@ Stage-0
| Select Operator [SEL_12]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_30]
+ | Filter Operator [FIL_32]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_11]
@@ -1985,10 +1985,10 @@ Stage-0
sort order:+
Statistics:Num rows: 4 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: float), _col3 (type: string), _col4 (type: int)
- Filter Operator [FIL_27]
+ Filter Operator [FIL_29]
predicate:((((_col1 + _col4) = 2) and _col0 is not null) and ((_col4 + 1) = 2)) (type: boolean)
Statistics:Num rows: 4 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_31]
+ Merge Join Operator [MERGEJOIN_33]
| condition map:[{"":"Outer Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2","_col3","_col4"]
@@ -2003,7 +2003,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_28]
+ | Filter Operator [FIL_30]
| predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -2019,7 +2019,7 @@ Stage-0
Select Operator [SEL_5]
outputColumnNames:["_col0","_col1"]
Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_29]
+ Filter Operator [FIL_31]
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_3]
@@ -2047,10 +2047,10 @@ Stage-0
Select Operator [SEL_13]
outputColumnNames:["_col0","_col1","_col2","_col3","_col4"]
Statistics:Num rows: 12 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_20]
+ Filter Operator [FIL_21]
predicate:((((_col4 + 1) = 2) and ((_col1 > 0) or (_col6 >= 0))) and ((_col1 + _col4) = 2)) (type: boolean)
Statistics:Num rows: 12 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_23]
+ Merge Join Operator [MERGEJOIN_24]
| condition map:[{"":"Right Outer Join0 to 1"},{"":"Right Outer Join0 to 2"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)","2":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4","_col6"]
@@ -2065,7 +2065,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_21]
+ | Filter Operator [FIL_22]
| predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -2081,7 +2081,7 @@ Stage-0
| Select Operator [SEL_5]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_22]
+ | Filter Operator [FIL_23]
| predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_3]
@@ -2418,10 +2418,10 @@ Stage-0
Select Operator [SEL_39]
outputColumnNames:["_col2","_col6"]
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_38]
+ Filter Operator [FIL_54]
predicate:((_col1 > 0) or (_col6 >= 0)) (type: boolean)
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_60]
+ Merge Join Operator [MERGEJOIN_62]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col6"]
@@ -2436,7 +2436,7 @@ Stage-0
| Select Operator [SEL_32]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_58]
+ | Filter Operator [FIL_60]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_31]
@@ -2452,10 +2452,10 @@ Stage-0
Select Operator [SEL_30]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_29]
+ Filter Operator [FIL_55]
predicate:((_col1 + _col4) >= 0) (type: boolean)
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_59]
+ Merge Join Operator [MERGEJOIN_61]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2","_col4"]
@@ -2467,7 +2467,7 @@ Stage-0
| sort order:+
| Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Filter Operator [FIL_56]
+ | Filter Operator [FIL_58]
| predicate:_col0 is not null (type: boolean)
| Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
| Limit [LIM_22]
@@ -2502,7 +2502,7 @@ Stage-0
| keys:key (type: string), c_int (type: int), c_float (type: float)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_57]
+ | Filter Operator [FIL_59]
| predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_12]
@@ -2515,7 +2515,7 @@ Stage-0
sort order:+
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Filter Operator [FIL_54]
+ Filter Operator [FIL_56]
predicate:_col0 is not null (type: boolean)
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
Limit [LIM_10]
@@ -2550,7 +2550,7 @@ Stage-0
keys:key (type: string), c_int (type: int), c_float (type: float)
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_55]
+ Filter Operator [FIL_57]
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
Statistics:Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_0]
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out b/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
index be58a2b..8398f68 100644
--- a/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
@@ -323,7 +323,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col22, _col26, _col50, _col58
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col0 = _col58) and (_col22 = _col26) and (_col50) IN ('KS', 'AL', 'MN', 'AL', 'SC', 'VT')) (type: boolean)
+ predicate: ((_col0 = _col58) and (_col22 = _col26)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col50 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
index c779368..38caa49 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
@@ -1924,7 +1924,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
#### A masked pattern was here ####
1000
-Warning: Shuffle Join MERGEJOIN[13][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[14][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: -- non-equi join
EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr)
PREHOOK: type: QUERY
@@ -2011,7 +2011,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join MERGEJOIN[13][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[14][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr)
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out b/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
index 288025d..4535058 100644
--- a/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
+++ b/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
@@ -320,7 +320,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col22, _col26, _col50, _col58
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col0 = _col58) and (_col22 = _col26) and (_col50) IN ('KS', 'AL', 'MN', 'AL', 'SC', 'VT')) (type: boolean)
+ predicate: ((_col0 = _col58) and (_col22 = _col26)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col50 (type: string)
[9/9] hive git commit: HIVE-12071 : LLAP: merge master into branch
(Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-12071 : LLAP: merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/be00a031
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/be00a031
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/be00a031
Branch: refs/heads/llap
Commit: be00a031da2d55ce8a6bf7280e2a797eb6741cd2
Parents: 7a23913 aded0d3
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 8 12:38:27 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 8 12:38:27 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hive/beeline/BeeLine.java | 4 +
.../org/apache/hive/beeline/BeeLineOpts.java | 9 +
.../hive/beeline/TestBeelineArgParsing.java | 1 +
.../org/apache/hadoop/hive/cli/CliDriver.java | 9 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../apache/hadoop/hive/ql/log/PerfLogger.java | 2 +-
.../apache/hadoop/hive/ql/exec/ColumnInfo.java | 2 +-
.../hive/ql/exec/tez/TezSessionState.java | 203 +++++++++++++++----
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 6 +-
.../ql/io/avro/AvroGenericRecordReader.java | 22 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 3 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 34 +++-
.../hadoop/hive/ql/ppd/OpProcFactory.java | 10 +-
.../hadoop/hive/ql/session/SessionState.java | 60 ++++--
.../hive/ql/udf/generic/GenericUDFUtils.java | 3 +-
.../hadoop/hive/ql/exec/tez/TestTezTask.java | 2 +
.../ql/io/avro/TestAvroGenericRecordReader.java | 59 ++++++
ql/src/test/queries/clientpositive/join44.q | 12 ++
ql/src/test/queries/clientpositive/temp_table.q | 26 +++
ql/src/test/queries/clientpositive/union36.q | 10 +
.../clientpositive/dynamic_rdd_cache.q.out | 28 +--
ql/src/test/results/clientpositive/join44.q.out | 88 ++++++++
.../join_cond_pushdown_unqual1.q.out | 12 +-
.../join_cond_pushdown_unqual3.q.out | 18 +-
.../join_cond_pushdown_unqual4.q.out | 2 +-
.../results/clientpositive/pointlookup2.q.out | 12 +-
.../results/clientpositive/pointlookup3.q.out | 12 +-
.../spark/dynamic_rdd_cache.q.out | 28 +--
.../spark/join_cond_pushdown_unqual1.q.out | 12 +-
.../spark/join_cond_pushdown_unqual3.q.out | 18 +-
.../spark/join_cond_pushdown_unqual4.q.out | 2 +-
.../clientpositive/spark/temp_table.q.out | 107 ++++++++++
.../results/clientpositive/temp_table.q.out | 107 ++++++++++
.../tez/dynamic_partition_pruning.q.out | 4 +-
.../tez/dynamic_partition_pruning_2.q.out | 54 ++---
.../clientpositive/tez/explainuser_1.q.out | 96 ++++-----
.../results/clientpositive/tez/temp_table.q.out | 107 ++++++++++
.../tez/vector_mr_diff_schema_alias.q.out | 2 +-
.../vectorized_dynamic_partition_pruning.q.out | 4 +-
.../test/results/clientpositive/union36.q.out | 28 +++
.../vector_mr_diff_schema_alias.q.out | 2 +-
42 files changed, 986 insertions(+), 239 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/be00a031/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/be00a031/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 41a742c,6ed6421..e034c71
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@@ -201,9 -211,7 +251,9 @@@ public class TezSessionState
// and finally we're ready to create and start the session
// generate basic tez config
- TezConfiguration tezConfig = new TezConfiguration(conf);
+ final TezConfiguration tezConfig = new TezConfiguration(conf);
+
+ // set up the staging directory to use
tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
Utilities.stripHivePasswordDetails(tezConfig);
@@@ -235,9 -223,8 +285,9 @@@
tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);
}
- session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig).setIsSession(true)
- .setLocalResources(commonLocalResources)
- final TezClient session = TezClient.create("HIVE-" + sessionId, tezConfig, true,
- commonLocalResources, null);
++ final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig)
++ .setIsSession(true).setLocalResources(commonLocalResources)
+ .setServicePluginDescriptor(servicePluginsDescriptor).build();
LOG.info("Opening new Tez Session (id: " + sessionId
+ ", scratch dir: " + tezScratchDir + ")");
http://git-wip-us.apache.org/repos/asf/hive/blob/be00a031/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/be00a031/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/be00a031/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
[7/9] hive git commit: HIVE-9695: Redundant filter operator in
reducer Vertex when CBO is disabled (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-9695: Redundant filter operator in reducer Vertex when CBO is disabled (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/48c420e5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/48c420e5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/48c420e5
Branch: refs/heads/llap
Commit: 48c420e5516156761f584797210e496e54f3f5d5
Parents: 768db67
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Oct 6 17:14:24 2015 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Thu Oct 8 13:33:10 2015 +0100
----------------------------------------------------------------------
.../hadoop/hive/ql/ppd/OpProcFactory.java | 10 +-
ql/src/test/queries/clientpositive/join44.q | 12 +++
.../clientpositive/dynamic_rdd_cache.q.out | 28 +++---
ql/src/test/results/clientpositive/join44.q.out | 88 ++++++++++++++++++
.../join_cond_pushdown_unqual1.q.out | 12 +--
.../join_cond_pushdown_unqual3.q.out | 18 ++--
.../join_cond_pushdown_unqual4.q.out | 2 +-
.../results/clientpositive/pointlookup2.q.out | 12 +--
.../results/clientpositive/pointlookup3.q.out | 12 +--
.../spark/dynamic_rdd_cache.q.out | 28 +++---
.../spark/join_cond_pushdown_unqual1.q.out | 12 +--
.../spark/join_cond_pushdown_unqual3.q.out | 18 ++--
.../spark/join_cond_pushdown_unqual4.q.out | 2 +-
.../tez/dynamic_partition_pruning.q.out | 4 +-
.../tez/dynamic_partition_pruning_2.q.out | 54 +++++------
.../clientpositive/tez/explainuser_1.q.out | 96 ++++++++++----------
.../tez/vector_mr_diff_schema_alias.q.out | 2 +-
.../vectorized_dynamic_partition_pruning.q.out | 4 +-
.../vector_mr_diff_schema_alias.q.out | 2 +-
19 files changed, 258 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
index dbd021b..8566374 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
@@ -424,15 +424,15 @@ public final class OpProcFactory {
}
return null;
}
+ logExpr(nd, ewi);
+ owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, ewi);
if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
// add this filter for deletion, if it does not have non-final candidates
- if (ewi.getNonFinalCandidates().values().isEmpty()) {
- owi.addCandidateFilterOp((FilterOperator)op);
- }
+ owi.addCandidateFilterOp((FilterOperator)op);
+ Map<String, List<ExprNodeDesc>> residual = ewi.getResidualPredicates(true);
+ createFilter(op, residual, owi);
}
- logExpr(nd, ewi);
- owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, ewi);
}
// merge it with children predicates
boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, ewi, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/queries/clientpositive/join44.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join44.q b/ql/src/test/queries/clientpositive/join44.q
new file mode 100644
index 0000000..0111079
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/join44.q
@@ -0,0 +1,12 @@
+set hive.cbo.enable=false;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE mytable(val1 INT, val2 INT, val3 INT);
+
+EXPLAIN
+SELECT *
+FROM mytable src1, mytable src2
+WHERE src1.val1=src2.val1
+ AND src1.val2 between 2450816 and 2451500
+ AND src2.val2 between 2450816 and 2451500;
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
index eeb5847..501a86f 100644
--- a/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
+++ b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
@@ -1030,7 +1030,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col11, _col12, _col16
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col1 = _col7) and (_col3 = _col11) and (_col0 = _col16)) (type: boolean)
+ predicate: (((_col1 = _col7) and (_col3 = _col11)) and (_col0 = _col16)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col12 (type: string), _col11 (type: int), _col7 (type: int), 3 (type: int), _col2 (type: int)
@@ -1067,15 +1067,15 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: double), _col5 (type: double)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double)
+ outputColumnNames: _col1, _col2, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (CASE (_col5) WHEN (0) THEN (0) ELSE ((_col4 / _col5)) END > 1) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
- outputColumnNames: _col1, _col2, _col3, _col5, _col6
+ expressions: _col1 (type: int), _col2 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
+ outputColumnNames: _col1, _col2, _col5, _col6
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: true
@@ -1093,14 +1093,14 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col2 (type: int), _col1 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: double), _col6 (type: double)
+ value expressions: _col5 (type: double), _col6 (type: double)
TableScan
Reduce Output Operator
key expressions: _col2 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col2 (type: int), _col1 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: double), _col6 (type: double)
+ value expressions: _col5 (type: double), _col6 (type: double)
Reduce Operator Tree:
Join Operator
condition map:
@@ -1108,10 +1108,10 @@ STAGE PLANS:
keys:
0 _col2 (type: int), _col1 (type: int)
1 _col2 (type: int), _col1 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col8, _col9, _col10, _col12, _col13
+ outputColumnNames: _col1, _col2, _col5, _col6, _col8, _col9, _col12, _col13
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col2 = _col9) and (_col1 = _col8) and (_col3 = 3) and (_col10 = 4)) (type: boolean)
+ predicate: ((_col2 = _col9) and (_col1 = _col8)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col1 (type: int), _col2 (type: int), _col5 (type: double), _col6 (type: double), _col8 (type: int), _col9 (type: int), _col12 (type: double), _col13 (type: double)
@@ -1257,7 +1257,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col11, _col12, _col16
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col1 = _col7) and (_col3 = _col11) and (_col0 = _col16)) (type: boolean)
+ predicate: (((_col1 = _col7) and (_col3 = _col11)) and (_col0 = _col16)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col12 (type: string), _col11 (type: int), _col7 (type: int), 4 (type: int), _col2 (type: int)
@@ -1294,15 +1294,15 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: double), _col5 (type: double)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double)
+ outputColumnNames: _col1, _col2, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (CASE (_col5) WHEN (0) THEN (0) ELSE ((_col4 / _col5)) END > 1) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
- outputColumnNames: _col1, _col2, _col3, _col5, _col6
+ expressions: _col1 (type: int), _col2 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
+ outputColumnNames: _col1, _col2, _col5, _col6
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: true
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/join44.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join44.q.out b/ql/src/test/results/clientpositive/join44.q.out
new file mode 100644
index 0000000..8598701
--- /dev/null
+++ b/ql/src/test/results/clientpositive/join44.q.out
@@ -0,0 +1,88 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE mytable(val1 INT, val2 INT, val3 INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mytable
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE mytable(val1 INT, val2 INT, val3 INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mytable
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM mytable src1, mytable src2
+WHERE src1.val1=src2.val1
+ AND src1.val2 between 2450816 and 2451500
+ AND src2.val2 between 2450816 and 2451500
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM mytable src1, mytable src2
+WHERE src1.val1=src2.val1
+ AND src1.val2 between 2450816 and 2451500
+ AND src2.val2 between 2450816 and 2451500
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (val1 is not null and val2 BETWEEN 2450816 AND 2451500) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: val1 (type: int)
+ sort order: +
+ Map-reduce partition columns: val1 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: val2 (type: int), val3 (type: int)
+ TableScan
+ alias: src2
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (val1 is not null and val2 BETWEEN 2450816 AND 2451500) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: val1 (type: int)
+ sort order: +
+ Map-reduce partition columns: val1 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: val2 (type: int), val3 (type: int)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 val1 (type: int)
+ 1 val1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (_col0 = _col6) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
index 597b75f..c1c2105 100644
--- a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
+++ b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
@@ -255,8 +255,8 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col12 + _col0) = _col0) and _col13 is not null) (type: boolean)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col12 + _col0) = _col0) (type: boolean)
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -272,7 +272,7 @@ STAGE PLANS:
key expressions: _col13 (type: string)
sort order: +
Map-reduce partition columns: _col13 (type: string)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string)
TableScan
alias: p3
@@ -294,14 +294,14 @@ STAGE PLANS:
0 _col13 (type: string)
1 p3_name (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
index 9b2da59..b0258b8 100644
--- a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
+++ b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
@@ -118,7 +118,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
Statistics: Num rows: 28 Data size: 3460 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col1 = _col13) and (_col13 = _col25)) (type: boolean)
+ predicate: ((_col13 = _col25) and (_col1 = _col13)) (type: boolean)
Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
@@ -202,7 +202,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
Statistics: Num rows: 28 Data size: 3460 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col13 = _col1) and (_col25 = _col13)) (type: boolean)
+ predicate: ((_col25 = _col13) and (_col13 = _col1)) (type: boolean)
Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
@@ -267,8 +267,8 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col12 + _col0) = _col0) and _col13 is not null) (type: boolean)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col12 + _col0) = _col0) (type: boolean)
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -284,7 +284,7 @@ STAGE PLANS:
key expressions: _col13 (type: string)
sort order: +
Map-reduce partition columns: _col13 (type: string)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string)
TableScan
alias: p3
@@ -306,17 +306,17 @@ STAGE PLANS:
0 _col13 (type: string)
1 p3_name (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (((_col12 + _col0) = _col0) and (_col25 = _col13)) (type: boolean)
- Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
- Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
index 6ff13e4..26db67e 100644
--- a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
+++ b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
@@ -282,7 +282,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44
Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col13 = _col25) and (_col0 = _col36) and (_col0 = _col12)) (type: boolean)
+ predicate: (((_col13 = _col25) and (_col0 = _col36)) and (_col0 = _col12)) (type: boolean)
Statistics: Num rows: 1 Data size: 123 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string), _col36 (type: int), _col37 (type: string), _col38 (type: string), _col39 (type: string), _col40 (type: string), _col41 (type: int), _col42 (type: string), _col43 (type: double), _col44 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/pointlookup2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/pointlookup2.q.out b/ql/src/test/results/clientpositive/pointlookup2.q.out
index 55edd90..700fbde 100644
--- a/ql/src/test/results/clientpositive/pointlookup2.q.out
+++ b/ql/src/test/results/clientpositive/pointlookup2.q.out
@@ -1128,12 +1128,12 @@ STAGE PLANS:
Statistics: Num rows: 44 Data size: 352 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col2) IN ('2000-04-08', '2000-04-09') and (struct(_col7,_col2)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09'))) (type: boolean)
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ predicate: (struct(_col7,_col2)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean)
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
@@ -1160,7 +1160,7 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col4 (type: int), _col5 (type: string), _col2 (type: string)
sort order: +++
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string)
auto parallelism: false
@@ -1194,13 +1194,13 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/pointlookup3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/pointlookup3.q.out b/ql/src/test/results/clientpositive/pointlookup3.q.out
index 4cfb97e..60a276b 100644
--- a/ql/src/test/results/clientpositive/pointlookup3.q.out
+++ b/ql/src/test/results/clientpositive/pointlookup3.q.out
@@ -1289,12 +1289,12 @@ STAGE PLANS:
Statistics: Num rows: 66 Data size: 528 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col2) IN ('2000-04-08', '2000-04-09') and (struct(_col7,_col2)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09'))) (type: boolean)
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ predicate: (struct(_col7,_col2)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean)
+ Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
@@ -1321,7 +1321,7 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col4 (type: int), _col5 (type: string), _col2 (type: string)
sort order: +++
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: string)
auto parallelism: false
@@ -1355,13 +1355,13 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out b/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out
index 7e9a0f3..730a31f 100644
--- a/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out
+++ b/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out
@@ -833,7 +833,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col11, _col12, _col16
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col1 = _col7) and (_col3 = _col11) and (_col0 = _col16)) (type: boolean)
+ predicate: (((_col1 = _col7) and (_col3 = _col11)) and (_col0 = _col16)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col12 (type: string), _col11 (type: int), _col7 (type: int), 4 (type: int), _col2 (type: int)
@@ -860,22 +860,22 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: double), _col5 (type: double)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double)
+ outputColumnNames: _col1, _col2, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (CASE (_col5) WHEN (0) THEN (0) ELSE ((_col4 / _col5)) END > 1) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
- outputColumnNames: _col1, _col2, _col3, _col5, _col6
+ expressions: _col1 (type: int), _col2 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
+ outputColumnNames: _col1, _col2, _col5, _col6
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col2 (type: int), _col1 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: double), _col6 (type: double)
+ value expressions: _col5 (type: double), _col6 (type: double)
Reducer 4
Reduce Operator Tree:
Join Operator
@@ -887,7 +887,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col11, _col12, _col16
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col1 = _col7) and (_col3 = _col11) and (_col0 = _col16)) (type: boolean)
+ predicate: (((_col1 = _col7) and (_col3 = _col11)) and (_col0 = _col16)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col12 (type: string), _col11 (type: int), _col7 (type: int), 3 (type: int), _col2 (type: int)
@@ -914,22 +914,22 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: double), _col5 (type: double)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double)
+ outputColumnNames: _col1, _col2, _col4, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (CASE (_col5) WHEN (0) THEN (0) ELSE ((_col4 / _col5)) END > 1) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
- outputColumnNames: _col1, _col2, _col3, _col5, _col6
+ expressions: _col1 (type: int), _col2 (type: int), _col5 (type: double), CASE (_col5) WHEN (0) THEN (null) ELSE ((_col4 / _col5)) END (type: double)
+ outputColumnNames: _col1, _col2, _col5, _col6
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col2 (type: int), _col1 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: double), _col6 (type: double)
+ value expressions: _col5 (type: double), _col6 (type: double)
Reducer 6
Reduce Operator Tree:
Join Operator
@@ -938,10 +938,10 @@ STAGE PLANS:
keys:
0 _col2 (type: int), _col1 (type: int)
1 _col2 (type: int), _col1 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col8, _col9, _col10, _col12, _col13
+ outputColumnNames: _col1, _col2, _col5, _col6, _col8, _col9, _col12, _col13
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((_col2 = _col9) and (_col1 = _col8) and (_col3 = 3) and (_col10 = 4)) (type: boolean)
+ predicate: ((_col2 = _col9) and (_col1 = _col8)) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col1 (type: int), _col2 (type: int), _col5 (type: double), _col6 (type: double), _col8 (type: int), _col9 (type: int), _col12 (type: double), _col13 (type: double)
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out b/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out
index 43955fa..5a77830 100644
--- a/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out
@@ -297,13 +297,13 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col12 + _col0) = _col0) and _col13 is not null) (type: boolean)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col12 + _col0) = _col0) (type: boolean)
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col13 (type: string)
sort order: +
Map-reduce partition columns: _col13 (type: string)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string)
Reducer 3
Reduce Operator Tree:
@@ -314,14 +314,14 @@ STAGE PLANS:
0 _col13 (type: string)
1 p3_name (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out b/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out
index e0df992..180787b 100644
--- a/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out
@@ -128,7 +128,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
Statistics: Num rows: 28 Data size: 3460 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col1 = _col13) and (_col13 = _col25)) (type: boolean)
+ predicate: ((_col13 = _col25) and (_col1 = _col13)) (type: boolean)
Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
@@ -222,7 +222,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
Statistics: Num rows: 28 Data size: 3460 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col13 = _col1) and (_col25 = _col13)) (type: boolean)
+ predicate: ((_col25 = _col13) and (_col13 = _col1)) (type: boolean)
Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
@@ -309,13 +309,13 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col12 + _col0) = _col0) and _col13 is not null) (type: boolean)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col12 + _col0) = _col0) (type: boolean)
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col13 (type: string)
sort order: +
Map-reduce partition columns: _col13 (type: string)
- Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string)
Reducer 3
Reduce Operator Tree:
@@ -326,17 +326,17 @@ STAGE PLANS:
0 _col13 (type: string)
1 p3_name (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32
- Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (((_col12 + _col0) = _col0) and (_col25 = _col13)) (type: boolean)
- Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
- Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual4.q.out b/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual4.q.out
index b30f4f4..e16884c 100644
--- a/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual4.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual4.q.out
@@ -286,7 +286,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44
Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col13 = _col25) and (_col0 = _col36) and (_col0 = _col12)) (type: boolean)
+ predicate: (((_col13 = _col25) and (_col0 = _col36)) and (_col0 = _col12)) (type: boolean)
Statistics: Num rows: 1 Data size: 123 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string), _col36 (type: int), _col37 (type: string), _col38 (type: string), _col39 (type: string), _col40 (type: string), _col41 (type: int), _col42 (type: string), _col43 (type: double), _col44 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
index bbc2e16..b6fa1ac 100644
--- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
@@ -1902,7 +1902,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
#### A masked pattern was here ####
1000
-Warning: Shuffle Join MERGEJOIN[13][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[14][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: -- non-equi join
EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr)
PREHOOK: type: QUERY
@@ -1988,7 +1988,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join MERGEJOIN[13][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[14][tables = [srcpart, srcpart_date_hour]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr)
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart
http://git-wip-us.apache.org/repos/asf/hive/blob/48c420e5/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
index c8e9da4..430d5ad 100644
--- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
@@ -178,23 +178,23 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 29 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Filter Operator
- predicate: ((_col1 = _col5) and (_col6) IN ('foo', 'bar')) (type: boolean)
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col1 = _col5) (type: boolean)
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col6 (type: string), _col0 (type: decimal(10,0))
outputColumnNames: _col6, _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
keys: _col6 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0))
Map 4
Map Operator Tree:
@@ -233,21 +233,21 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0))
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -340,23 +340,23 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 29 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Filter Operator
- predicate: ((_col1 = _col5) and (_col6) IN ('foo', 'bar')) (type: boolean)
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col1 = _col5) (type: boolean)
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col6 (type: string), _col0 (type: decimal(10,0))
outputColumnNames: _col6, _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
keys: _col6 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0))
Map 4
Map Operator Tree:
@@ -380,21 +380,21 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0))
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -683,23 +683,23 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 29 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Filter Operator
- predicate: ((_col1 = _col5) and (_col6) IN ('foo', 'bar')) (type: boolean)
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col1 = _col5) (type: boolean)
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col6 (type: string), _col0 (type: decimal(10,0))
outputColumnNames: _col6, _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
keys: _col6 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0))
Map 4
Map Operator Tree:
@@ -738,21 +738,21 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0))
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
[5/9] hive git commit: HIVE-11919: Hive Union Type Mismatch (Laljo
John Pullokkaran, reviewed by Jesus Camacho Rodriguez)
Posted by se...@apache.org.
HIVE-11919: Hive Union Type Mismatch (Laljo John Pullokkaran, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/768db673
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/768db673
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/768db673
Branch: refs/heads/llap
Commit: 768db6732031a3a41cee9996a1021621ce244100
Parents: 7f9023e
Author: jpullokk <jp...@apache.org>
Authored: Wed Oct 7 21:12:51 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Wed Oct 7 21:12:51 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/ColumnInfo.java | 2 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 3 +--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +--
.../hive/ql/udf/generic/GenericUDFUtils.java | 3 +--
ql/src/test/queries/clientpositive/union36.q | 10 +++++++
.../test/results/clientpositive/union36.q.out | 28 ++++++++++++++++++++
6 files changed, 42 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/768db673/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
index 12bb1d7..e3da7f0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
@@ -107,7 +107,6 @@ public class ColumnInfo implements Serializable {
this.isVirtualCol = columnInfo.getIsVirtualCol();
this.isHiddenVirtualCol = columnInfo.isHiddenVirtualCol();
this.setType(columnInfo.getType());
- this.typeName = columnInfo.getType().getTypeName();
}
public String getTypeName() {
@@ -133,6 +132,7 @@ public class ColumnInfo implements Serializable {
public void setType(TypeInfo type) {
objectInspector =
TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(type);
+ setTypeName(type.getTypeName());
}
public void setInternalName(String internalName) {
http://git-wip-us.apache.org/repos/asf/hive/blob/768db673/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 9c731b8..e68b385 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1124,8 +1124,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
+ " on second table"));
}
ColumnInfo unionColInfo = new ColumnInfo(lInfo);
- unionColInfo.setType(FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(),
- rInfo.getType()));
+ unionColInfo.setType(commonTypeInfo);
unionoutRR.put(unionalias, field, unionColInfo);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/768db673/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 7a54aec..a114281 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -9026,8 +9026,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
+ " on second table"));
}
ColumnInfo unionColInfo = new ColumnInfo(lInfo);
- unionColInfo.setType(FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(),
- rInfo.getType()));
+ unionColInfo.setType(commonTypeInfo);
unionoutRR.put(unionalias, field, unionColInfo);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/768db673/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
index 222e0e0..3bbe783 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
@@ -159,8 +159,7 @@ public final class GenericUDFUtils {
// a common base class or not.
TypeInfo commonTypeInfo = null;
if (isUnionAll) {
- commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(oiTypeInfo,
- rTypeInfo);
+ commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(rTypeInfo, oiTypeInfo);
} else {
commonTypeInfo = FunctionRegistry.getCommonClass(oiTypeInfo,
rTypeInfo);
http://git-wip-us.apache.org/repos/asf/hive/blob/768db673/ql/src/test/queries/clientpositive/union36.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/union36.q b/ql/src/test/queries/clientpositive/union36.q
new file mode 100644
index 0000000..e929749
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/union36.q
@@ -0,0 +1,10 @@
+set hive.cbo.enable=false;
+
+select (x/sum(x) over()) as y from(select cast(1 as decimal(10,0)) as x from (select * from src limit 2)s1 union all select cast(1 as decimal(10,0)) x from (select * from src limit 2) s2 union all select '100000000' x from (select * from src limit 2) s3)u order by y;
+
+select (x/sum(x) over()) as y from(select cast(1 as decimal(10,0)) as x from (select * from src limit 2)s1 union all select cast(1 as decimal(10,0)) x from (select * from src limit 2) s2 union all select cast (null as string) x from (select * from src limit 2) s3)u order by y;
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/768db673/ql/src/test/results/clientpositive/union36.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union36.q.out b/ql/src/test/results/clientpositive/union36.q.out
new file mode 100644
index 0000000..12f060b
--- /dev/null
+++ b/ql/src/test/results/clientpositive/union36.q.out
@@ -0,0 +1,28 @@
+PREHOOK: query: select (x/sum(x) over()) as y from(select cast(1 as decimal(10,0)) as x from (select * from src limit 2)s1 union all select cast(1 as decimal(10,0)) x from (select * from src limit 2) s2 union all select '100000000' x from (select * from src limit 2) s3)u order by y
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select (x/sum(x) over()) as y from(select cast(1 as decimal(10,0)) as x from (select * from src limit 2)s1 union all select cast(1 as decimal(10,0)) x from (select * from src limit 2) s2 union all select '100000000' x from (select * from src limit 2) s3)u order by y
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+4.999999900000002E-9
+4.999999900000002E-9
+4.999999900000002E-9
+4.999999900000002E-9
+0.4999999900000002
+0.4999999900000002
+PREHOOK: query: select (x/sum(x) over()) as y from(select cast(1 as decimal(10,0)) as x from (select * from src limit 2)s1 union all select cast(1 as decimal(10,0)) x from (select * from src limit 2) s2 union all select cast (null as string) x from (select * from src limit 2) s3)u order by y
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select (x/sum(x) over()) as y from(select cast(1 as decimal(10,0)) as x from (select * from src limit 2)s1 union all select cast(1 as decimal(10,0)) x from (select * from src limit 2) s2 union all select cast (null as string) x from (select * from src limit 2) s3)u order by y
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+NULL
+NULL
+0.25
+0.25
+0.25
+0.25
[4/9] hive git commit: HIVE-11969 : start Tez session in background
when starting CLI (Sergey Shelukhin, reviewed by Gopal V)
Posted by se...@apache.org.
HIVE-11969 : start Tez session in background when starting CLI (Sergey Shelukhin, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7f9023ea
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7f9023ea
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7f9023ea
Branch: refs/heads/llap
Commit: 7f9023ea0323821626f17e30d04b5acffb1d3048
Parents: 556877c
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Oct 7 13:55:36 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Oct 7 13:55:36 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/cli/CliDriver.java | 9 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../hive/ql/exec/tez/TezSessionState.java | 204 +++++++++++++++----
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 6 +-
.../hadoop/hive/ql/session/SessionState.java | 52 +++--
.../hadoop/hive/ql/exec/tez/TestTezTask.java | 2 +
6 files changed, 222 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7f9023ea/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
----------------------------------------------------------------------
diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
index 4b52578..3a80f99 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.Validator;
import org.apache.hadoop.hive.conf.VariableSubstitution;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
@@ -690,7 +691,13 @@ public class CliDriver {
}).substitute(conf, prompt);
prompt2 = spacesForString(prompt);
- SessionState.start(ss);
+ if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CLI_TEZ_SESSION_ASYNC)) {
+ // Start the session in a fire-and-forget manner. When the asynchronously initialized parts of
+ // the session are needed, the corresponding getters and other methods will wait as needed.
+ SessionState.beginStart(ss, console);
+ } else {
+ SessionState.start(ss);
+ }
// execute cli driver work
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/7f9023ea/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 54a529e..bf48f69 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1712,6 +1712,9 @@ public class HiveConf extends Configuration {
HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false, "Whether to print the names of the columns in query output."),
+ HIVE_CLI_TEZ_SESSION_ASYNC("hive.cli.tez.session.async", true, "Whether to start Tez\n" +
+ "session in background when running CLI with Tez, allowing CLI to be available earlier."),
+
HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false,
"Whether to throw an exception if dynamic partition insert generates empty results."),
http://git-wip-us.apache.org/repos/asf/hive/blob/7f9023ea/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 568ebbe..6ed6421 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -23,14 +23,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import javax.security.auth.login.LoginException;
@@ -46,7 +51,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -70,6 +75,9 @@ public class TezSessionState {
private Path tezScratchDir;
private LocalResource appJarLr;
private TezClient session;
+ private Future<TezClient> sessionFuture;
+ /** Console used for user feedback during async session opening. */
+ private LogHelper console;
private String sessionId;
private final DagUtils utils;
private String queueName;
@@ -97,13 +105,40 @@ public class TezSessionState {
this.sessionId = sessionId;
}
- /**
- * Returns whether a session has been established
- */
+ public boolean isOpening() {
+ if (session != null || sessionFuture == null) return false;
+ try {
+ session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ } catch (CancellationException e) {
+ return false;
+ } catch (TimeoutException e) {
+ return true;
+ }
+ return false;
+ }
+
public boolean isOpen() {
- return session != null;
+ if (session != null) return true;
+ if (sessionFuture == null) return false;
+ try {
+ session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ } catch (TimeoutException | CancellationException e) {
+ return false;
+ }
+ return true;
}
+
/**
* Get all open sessions. Only used to clean up at shutdown.
* @return List<TezSessionState>
@@ -124,9 +159,21 @@ public class TezSessionState {
* @throws URISyntaxException
* @throws LoginException
* @throws TezException
+ * @throws InterruptedException
*/
public void open(HiveConf conf, String[] additionalFiles)
throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
+ openInternal(conf, additionalFiles, false, null);
+ }
+
+ public void beginOpen(HiveConf conf, String[] additionalFiles, LogHelper console)
+ throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
+ openInternal(conf, additionalFiles, true, console);
+ }
+
+ private void openInternal(
+ final HiveConf conf, String[] additionalFiles, boolean isAsync, LogHelper console)
+ throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
this.conf = conf;
this.queueName = conf.get("tez.queue.name");
this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS);
@@ -152,7 +199,7 @@ public class TezSessionState {
appJarLr = createJarLocalResource(utils.getExecJarPathLocal());
// configuration for the application master
- Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
+ final Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
commonLocalResources.put(utils.getBaseName(appJarLr), appJarLr);
for (LocalResource lr : localizedResources) {
commonLocalResources.put(utils.getBaseName(lr), lr);
@@ -164,7 +211,7 @@ public class TezSessionState {
// and finally we're ready to create and start the session
// generate basic tez config
- TezConfiguration tezConfig = new TezConfiguration(conf);
+ final TezConfiguration tezConfig = new TezConfiguration(conf);
tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
Utilities.stripHivePasswordDetails(tezConfig);
@@ -176,37 +223,85 @@ public class TezSessionState {
tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);
}
- session = TezClient.create("HIVE-" + sessionId, tezConfig, true,
+ final TezClient session = TezClient.create("HIVE-" + sessionId, tezConfig, true,
commonLocalResources, null);
LOG.info("Opening new Tez Session (id: " + sessionId
+ ", scratch dir: " + tezScratchDir + ")");
TezJobMonitor.initShutdownHook();
- session.start();
+ if (!isAsync) {
+ startSessionAndContainers(session, conf, commonLocalResources, tezConfig, false);
+ this.session = session;
+ } else {
+ FutureTask<TezClient> sessionFuture = new FutureTask<>(new Callable<TezClient>() {
+ @Override
+ public TezClient call() throws Exception {
+ return startSessionAndContainers(session, conf, commonLocalResources, tezConfig, true);
+ }
+ });
+ new Thread(sessionFuture, "Tez session start thread").start();
+ // We assume here nobody will try to get session before open() returns.
+ this.console = console;
+ this.sessionFuture = sessionFuture;
+ }
+ }
- if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
- int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
- LOG.info("Prewarming " + n + " containers (id: " + sessionId
- + ", scratch dir: " + tezScratchDir + ")");
- PreWarmVertex prewarmVertex = utils.createPreWarmVertex(tezConfig, n,
- commonLocalResources);
- try {
- session.preWarm(prewarmVertex);
- } catch (IOException ie) {
- if (ie.getMessage().contains("Interrupted while waiting")) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Hive Prewarm threw an exception ", ie);
+ private TezClient startSessionAndContainers(TezClient session, HiveConf conf,
+ Map<String, LocalResource> commonLocalResources, TezConfiguration tezConfig,
+ boolean isOnThread) throws TezException, IOException {
+ session.start();
+ boolean isSuccessful = false;
+ try {
+ if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
+ int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
+ LOG.info("Prewarming " + n + " containers (id: " + sessionId
+ + ", scratch dir: " + tezScratchDir + ")");
+ PreWarmVertex prewarmVertex = utils.createPreWarmVertex(
+ tezConfig, n, commonLocalResources);
+ try {
+ session.preWarm(prewarmVertex);
+ } catch (IOException ie) {
+ if (!isOnThread && ie.getMessage().contains("Interrupted while waiting")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Hive Prewarm threw an exception ", ie);
+ }
+ } else {
+ throw ie;
}
- } else {
- throw ie;
}
}
+ try {
+ session.waitTillReady();
+ } catch (InterruptedException ie) {
+ if (isOnThread) throw new IOException(ie);
+ //ignore
+ }
+ isSuccessful = true;
+ return session;
+ } finally {
+ if (isOnThread && !isSuccessful) {
+ closeAndIgnoreExceptions(session);
+ }
}
+ }
+
+ private static void closeAndIgnoreExceptions(TezClient session) {
+ try {
+ session.stop();
+ } catch (SessionNotRunning nr) {
+ // Ignore.
+ } catch (IOException | TezException ex) {
+ LOG.info("Failed to close Tez session after failure to initialize: " + ex.getMessage());
+ }
+ }
+
+ public void endOpen() throws InterruptedException, CancellationException {
+ if (this.session != null || this.sessionFuture == null) return;
try {
- session.waitTillReady();
- } catch(InterruptedException ie) {
- //ignore
+ this.session = this.sessionFuture.get();
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
}
}
@@ -250,21 +345,32 @@ public class TezSessionState {
* @throws Exception
*/
public void close(boolean keepTmpDir) throws Exception {
- if (!isOpen()) {
- return;
- }
-
- LOG.info("Closing Tez Session");
- try {
- session.stop();
- } catch (SessionNotRunning nr) {
- // ignore
+ if (session != null) {
+ LOG.info("Closing Tez Session");
+ closeClient(session);
+ } else if (sessionFuture != null) {
+ sessionFuture.cancel(true);
+ TezClient asyncSession = null;
+ try {
+ asyncSession = sessionFuture.get(); // In case it was done and noone looked at it.
+ } catch (ExecutionException | CancellationException e) {
+ // ignore
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ // ignore
+ }
+ if (asyncSession != null) {
+ LOG.info("Closing Tez Session");
+ closeClient(asyncSession);
+ }
}
if (!keepTmpDir) {
cleanupScratchDir();
}
session = null;
+ sessionFuture = null;
+ console = null;
tezScratchDir = null;
conf = null;
appJarLr = null;
@@ -272,6 +378,15 @@ public class TezSessionState {
localizedResources.clear();
}
+ private void closeClient(TezClient client) throws TezException,
+ IOException {
+ try {
+ client.stop();
+ } catch (SessionNotRunning nr) {
+ // ignore
+ }
+ }
+
public void cleanupScratchDir () throws IOException {
FileSystem fs = tezScratchDir.getFileSystem(conf);
fs.delete(tezScratchDir, true);
@@ -283,6 +398,21 @@ public class TezSessionState {
}
public TezClient getSession() {
+ if (session == null && sessionFuture != null) {
+ if (!sessionFuture.isDone()) {
+ console.printInfo("Waiting for Tez session and AM to be ready...");
+ }
+ try {
+ session = sessionFuture.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return null;
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ } catch (CancellationException e) {
+ return null;
+ }
+ }
return session;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7f9023ea/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 2d740ed..c62e929 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.tez.client.TezClient;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
@@ -251,7 +252,8 @@ public class TezTask extends Task<TezWork> {
final boolean missingLocalResources = !session
.hasResources(inputOutputJars);
- if (!session.isOpen()) {
+ TezClient client = session.getSession();
+ if (client == null) {
// can happen if the user sets the tez flag after the session was
// established
LOG.info("Tez session hasn't been created yet. Opening session");
@@ -263,7 +265,7 @@ public class TezTask extends Task<TezWork> {
if (missingLocalResources) {
LOG.info("Tez session missing resources," +
" adding additional necessary resources");
- session.getSession().addAppMasterLocalFiles(extraResources);
+ client.addAppMasterLocalFiles(extraResources);
}
session.refreshLocalResourcesFromConf(conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/7f9023ea/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index dc8c336..56b0fae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.io.FileUtils;
@@ -85,7 +86,6 @@ import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Shell;
import com.google.common.base.Preconditions;
@@ -474,6 +474,21 @@ public class SessionState {
* when switching from one session to another.
*/
public static SessionState start(SessionState startSs) {
+ start(startSs, false, null);
+ return startSs;
+ }
+
+ public static void beginStart(SessionState startSs, LogHelper console) {
+ start(startSs, true, console);
+ }
+
+ public static void endStart(SessionState startSs)
+ throws CancellationException, InterruptedException {
+ if (startSs.tezSessionState == null) return;
+ startSs.tezSessionState.endOpen();
+ }
+
+ private static void start(SessionState startSs, boolean isAsync, LogHelper console) {
setCurrentSessionState(startSs);
if (startSs.hiveHist == null){
@@ -521,20 +536,31 @@ public class SessionState {
throw new RuntimeException(e);
}
- if (HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)
- .equals("tez") && (startSs.isHiveServerQuery == false)) {
- try {
- if (startSs.tezSessionState == null) {
- startSs.tezSessionState = new TezSessionState(startSs.getSessionId());
- }
- if (!startSs.tezSessionState.isOpen()) {
- startSs.tezSessionState.open(startSs.conf); // should use conf on session start-up
+ String engine = HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
+ if (!engine.equals("tez") || startSs.isHiveServerQuery) return;
+
+ try {
+ if (startSs.tezSessionState == null) {
+ startSs.tezSessionState = new TezSessionState(startSs.getSessionId());
+ }
+ if (startSs.tezSessionState.isOpen()) {
+ return;
+ }
+ if (startSs.tezSessionState.isOpening()) {
+ if (!isAsync) {
+ startSs.tezSessionState.endOpen();
}
- } catch (Exception e) {
- throw new RuntimeException(e);
+ return;
+ }
+ // Neither open nor opening.
+ if (!isAsync) {
+ startSs.tezSessionState.open(startSs.conf); // should use conf on session start-up
+ } else {
+ startSs.tezSessionState.beginOpen(startSs.conf, null, console);
}
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
- return startSs;
}
/**
@@ -1572,8 +1598,6 @@ public class SessionState {
}
}
-
-
public TezSessionState getTezSession() {
return tezSessionState;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7f9023ea/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
index d004a27..858cca0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
@@ -236,6 +236,7 @@ public class TestTezTask {
.thenReturn(resources);
when(utils.getBaseName(res)).thenReturn("foo.jar");
when(sessionState.isOpen()).thenReturn(true);
+ when(sessionState.isOpening()).thenReturn(false);
when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
task.updateSession(sessionState, conf, path, inputOutputJars, resMap);
verify(session).addAppMasterLocalFiles(resMap);
@@ -254,6 +255,7 @@ public class TestTezTask {
.thenReturn(resources);
when(utils.getBaseName(res)).thenReturn("foo.jar");
when(sessionState.isOpen()).thenReturn(true);
+ when(sessionState.isOpening()).thenReturn(false);
when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
task.addExtraResourcesToDag(sessionState, dag, inputOutputJars, resMap);
verify(dag).addTaskLocalFiles(resMap);