You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/09/28 21:03:37 UTC
[1/7] hive git commit: HIVE-14090: JDOExceptions thrown by the
Metastore have their full stack trace returned to clients (Sahil Takiar,
reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 429a2da20 -> e045c5a57
HIVE-14090: JDOExceptions thrown by the Metastore have their full stack trace returned to clients (Sahil Takiar, reviewed by Sergio Pena)
Change-Id: I5242436f65c2cc2c9e48f7ebca518f1930e5f4c1
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3701a7aa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3701a7aa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3701a7aa
Branch: refs/heads/branch-2.1
Commit: 3701a7aa2e5e75b3a4d794618ee74d693963d30b
Parents: 429a2da
Author: Sahil Takiar <ta...@gmail.com>
Authored: Wed Jul 6 16:34:19 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:36:09 2016 -0500
----------------------------------------------------------------------
.../apache/hadoop/hive/metastore/RetryingHMSHandler.java | 6 +++---
.../create_with_constraints_duplicate_name.q.out | 11 +----------
2 files changed, 4 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3701a7aa/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
index 2fc487f..e46b50d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
@@ -200,9 +200,9 @@ public class RetryingHMSHandler implements InvocationHandler {
if (retryCount >= retryLimit) {
LOG.error("HMSHandler Fatal error: " + ExceptionUtils.getStackTrace(caughtException));
- // Since returning exceptions with a nested "cause" can be a problem in
- // Thrift, we are stuffing the stack trace into the message itself.
- throw new MetaException(ExceptionUtils.getStackTrace(caughtException));
+ MetaException me = new MetaException(caughtException.getMessage());
+ me.initCause(caughtException);
+ throw me;
}
assert (retryInterval >= 0);
http://git-wip-us.apache.org/repos/asf/hive/blob/3701a7aa/ql/src/test/results/clientnegative/create_with_constraints_duplicate_name.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/create_with_constraints_duplicate_name.q.out b/ql/src/test/results/clientnegative/create_with_constraints_duplicate_name.q.out
index 8d6801e..989ff38 100644
--- a/ql/src/test/results/clientnegative/create_with_constraints_duplicate_name.q.out
+++ b/ql/src/test/results/clientnegative/create_with_constraints_duplicate_name.q.out
@@ -10,13 +10,4 @@ PREHOOK: query: create table t2(x int, constraint pk1 primary key (x) disable no
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t2
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:javax.jdo.JDOUserException: One or more instances could not be made persistent
-#### A masked pattern was here ####
-NestedThrowablesStackTrace:
-Insert of object "#### A masked pattern was here ####" using statement "INSERT INTO KEY_CONSTRAINTS (CHILD_CD_ID,CHILD_INTEGER_IDX,CHILD_TBL_ID,CONSTRAINT_TYPE,DELETE_RULE,ENABLE_VALIDATE_RELY,PARENT_CD_ID,PARENT_INTEGER_IDX,PARENT_TBL_ID,UPDATE_RULE,"CONSTRAINT_NAME","POSITION") VALUES (?,?,?,?,?,?,?,?,?,?,?,?)" failed : The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by 'KEY_CONSTRAINTS_PK' defined on 'KEY_CONSTRAINTS'.
-org.datanucleus.exceptions.NucleusDataStoreException: Insert of object "#### A masked pattern was here ####" using statement "INSERT INTO KEY_CONSTRAINTS (CHILD_CD_ID,CHILD_INTEGER_IDX,CHILD_TBL_ID,CONSTRAINT_TYPE,DELETE_RULE,ENABLE_VALIDATE_RELY,PARENT_CD_ID,PARENT_INTEGER_IDX,PARENT_TBL_ID,UPDATE_RULE,"CONSTRAINT_NAME","POSITION") VALUES (?,?,?,?,?,?,?,?,?,?,?,?)" failed : The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by 'KEY_CONSTRAINTS_PK' defined on 'KEY_CONSTRAINTS'.
-#### A masked pattern was here ####
-Nested Throwables StackTrace:
-java.sql.SQLIntegrityConstraintViolationException: The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by 'KEY_CONSTRAINTS_PK' defined on 'KEY_CONSTRAINTS'.
-#### A masked pattern was here ####
-)
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:One or more instances could not be made persistent)
[6/7] hive git commit: HIVE-14743: ArrayIndexOutOfBoundsException -
HBASE-backed views' query with JOINs (Yongzhi Chen, reviewed by Aihua Xu)
Posted by sp...@apache.org.
HIVE-14743: ArrayIndexOutOfBoundsException - HBASE-backed views' query with JOINs (Yongzhi Chen, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fb99e3e6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fb99e3e6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fb99e3e6
Branch: refs/heads/branch-2.1
Commit: fb99e3e60941326615bd210ac9057e33481a79aa
Parents: 7db2ec5
Author: Yongzhi Chen <yc...@apache.org>
Authored: Tue Sep 13 13:05:13 2016 -0400
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:54:14 2016 -0500
----------------------------------------------------------------------
.../hive/hbase/HiveHBaseTableInputFormat.java | 5 +-
.../src/test/queries/positive/hbase_viewjoins.q | 56 ++++++
.../test/results/positive/hbase_viewjoins.q.out | 172 +++++++++++++++++++
3 files changed, 232 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fb99e3e6/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
index d9db624..1ef4545 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.FloatWritable;
@@ -221,7 +223,8 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
SerializationUtilities.deserializeExpression(filterExprSerialized);
String keyColName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iKey];
- String colType = jobConf.get(serdeConstants.LIST_COLUMN_TYPES).split(",")[iKey];
+ ArrayList<TypeInfo> cols = TypeInfoUtils.getTypeInfosFromTypeString(jobConf.get(serdeConstants.LIST_COLUMN_TYPES));
+ String colType = cols.get(iKey).getTypeName();
boolean isKeyComparable = isKeyBinary || colType.equalsIgnoreCase("string");
String tsColName = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/fb99e3e6/hbase-handler/src/test/queries/positive/hbase_viewjoins.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_viewjoins.q b/hbase-handler/src/test/queries/positive/hbase_viewjoins.q
new file mode 100644
index 0000000..5c98903
--- /dev/null
+++ b/hbase-handler/src/test/queries/positive/hbase_viewjoins.q
@@ -0,0 +1,56 @@
+DROP VIEW IF EXISTS VIEW_HBASE_TABLE_TEST_2;
+DROP VIEW IF EXISTS VIEW_HBASE_TABLE_TEST_1;
+DROP TABLE IF EXISTS HBASE_TABLE_TEST_2;
+DROP TABLE IF EXISTS HBASE_TABLE_TEST_1;
+CREATE TABLE HBASE_TABLE_TEST_1(
+ cvalue string ,
+ pk string,
+ ccount int )
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.hbase.HBaseSerDe'
+STORED BY
+ 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+ 'hbase.columns.mapping'='cf:val,:key,cf2:count',
+ 'hbase.scan.cache'='500',
+ 'hbase.scan.cacheblocks'='false',
+ 'serialization.format'='1')
+TBLPROPERTIES (
+ 'hbase.table.name'='hbase_table_test_1',
+ 'serialization.null.format'='' );
+
+CREATE VIEW VIEW_HBASE_TABLE_TEST_1 AS SELECT hbase_table_test_1.cvalue,hbase_table_test_1.pk,hbase_table_test_1.ccount FROM hbase_table_test_1 WHERE hbase_table_test_1.ccount IS NOT NULL;
+
+CREATE TABLE HBASE_TABLE_TEST_2(
+ cvalue string ,
+ pk string ,
+ ccount int )
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.hbase.HBaseSerDe'
+STORED BY
+ 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+ 'hbase.columns.mapping'='cf:val,:key,cf2:count',
+ 'hbase.scan.cache'='500',
+ 'hbase.scan.cacheblocks'='false',
+ 'serialization.format'='1')
+TBLPROPERTIES (
+ 'hbase.table.name'='hbase_table_test_2',
+ 'serialization.null.format'='');
+
+CREATE VIEW VIEW_HBASE_TABLE_TEST_2 AS SELECT hbase_table_test_2.cvalue,hbase_table_test_2.pk,hbase_table_test_2.ccount
+FROM hbase_table_test_2 WHERE hbase_table_test_2.pk >='3-0000h-0' AND hbase_table_test_2.pk <= '3-0000h-g' AND
+hbase_table_test_2.ccount IS NOT NULL;
+
+set hive.auto.convert.join=false;
+
+SELECT p.cvalue cvalue
+FROM `VIEW_HBASE_TABLE_TEST_1` `p`
+LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_2` `A1`
+ON `p`.cvalue = `A1`.cvalue
+LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_1` `A2`
+ON `p`.cvalue = `A2`.cvalue;
+DROP VIEW VIEW_HBASE_TABLE_TEST_2;
+DROP VIEW VIEW_HBASE_TABLE_TEST_1;
+DROP TABLE HBASE_TABLE_TEST_2;
+DROP TABLE HBASE_TABLE_TEST_1;
http://git-wip-us.apache.org/repos/asf/hive/blob/fb99e3e6/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out b/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out
new file mode 100644
index 0000000..908024c
--- /dev/null
+++ b/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out
@@ -0,0 +1,172 @@
+PREHOOK: query: DROP VIEW IF EXISTS VIEW_HBASE_TABLE_TEST_2
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW IF EXISTS VIEW_HBASE_TABLE_TEST_2
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW IF EXISTS VIEW_HBASE_TABLE_TEST_1
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW IF EXISTS VIEW_HBASE_TABLE_TEST_1
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP TABLE IF EXISTS HBASE_TABLE_TEST_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS HBASE_TABLE_TEST_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS HBASE_TABLE_TEST_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS HBASE_TABLE_TEST_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE HBASE_TABLE_TEST_1(
+ cvalue string ,
+ pk string,
+ ccount int )
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.hbase.HBaseSerDe'
+STORED BY
+ 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+ 'hbase.columns.mapping'='cf:val,:key,cf2:count',
+ 'hbase.scan.cache'='500',
+ 'hbase.scan.cacheblocks'='false',
+ 'serialization.format'='1')
+TBLPROPERTIES (
+ 'hbase.table.name'='hbase_table_test_1',
+ 'serialization.null.format'='' )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@HBASE_TABLE_TEST_1
+POSTHOOK: query: CREATE TABLE HBASE_TABLE_TEST_1(
+ cvalue string ,
+ pk string,
+ ccount int )
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.hbase.HBaseSerDe'
+STORED BY
+ 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+ 'hbase.columns.mapping'='cf:val,:key,cf2:count',
+ 'hbase.scan.cache'='500',
+ 'hbase.scan.cacheblocks'='false',
+ 'serialization.format'='1')
+TBLPROPERTIES (
+ 'hbase.table.name'='hbase_table_test_1',
+ 'serialization.null.format'='' )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@HBASE_TABLE_TEST_1
+PREHOOK: query: CREATE VIEW VIEW_HBASE_TABLE_TEST_1 AS SELECT hbase_table_test_1.cvalue,hbase_table_test_1.pk,hbase_table_test_1.ccount FROM hbase_table_test_1 WHERE hbase_table_test_1.ccount IS NOT NULL
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@hbase_table_test_1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@VIEW_HBASE_TABLE_TEST_1
+POSTHOOK: query: CREATE VIEW VIEW_HBASE_TABLE_TEST_1 AS SELECT hbase_table_test_1.cvalue,hbase_table_test_1.pk,hbase_table_test_1.ccount FROM hbase_table_test_1 WHERE hbase_table_test_1.ccount IS NOT NULL
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@hbase_table_test_1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@VIEW_HBASE_TABLE_TEST_1
+PREHOOK: query: CREATE TABLE HBASE_TABLE_TEST_2(
+ cvalue string ,
+ pk string ,
+ ccount int )
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.hbase.HBaseSerDe'
+STORED BY
+ 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+ 'hbase.columns.mapping'='cf:val,:key,cf2:count',
+ 'hbase.scan.cache'='500',
+ 'hbase.scan.cacheblocks'='false',
+ 'serialization.format'='1')
+TBLPROPERTIES (
+ 'hbase.table.name'='hbase_table_test_2',
+ 'serialization.null.format'='')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@HBASE_TABLE_TEST_2
+POSTHOOK: query: CREATE TABLE HBASE_TABLE_TEST_2(
+ cvalue string ,
+ pk string ,
+ ccount int )
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.hbase.HBaseSerDe'
+STORED BY
+ 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+ 'hbase.columns.mapping'='cf:val,:key,cf2:count',
+ 'hbase.scan.cache'='500',
+ 'hbase.scan.cacheblocks'='false',
+ 'serialization.format'='1')
+TBLPROPERTIES (
+ 'hbase.table.name'='hbase_table_test_2',
+ 'serialization.null.format'='')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@HBASE_TABLE_TEST_2
+PREHOOK: query: CREATE VIEW VIEW_HBASE_TABLE_TEST_2 AS SELECT hbase_table_test_2.cvalue,hbase_table_test_2.pk,hbase_table_test_2.ccount
+FROM hbase_table_test_2 WHERE hbase_table_test_2.pk >='3-0000h-0' AND hbase_table_test_2.pk <= '3-0000h-g' AND
+hbase_table_test_2.ccount IS NOT NULL
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@hbase_table_test_2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@VIEW_HBASE_TABLE_TEST_2
+POSTHOOK: query: CREATE VIEW VIEW_HBASE_TABLE_TEST_2 AS SELECT hbase_table_test_2.cvalue,hbase_table_test_2.pk,hbase_table_test_2.ccount
+FROM hbase_table_test_2 WHERE hbase_table_test_2.pk >='3-0000h-0' AND hbase_table_test_2.pk <= '3-0000h-g' AND
+hbase_table_test_2.ccount IS NOT NULL
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@hbase_table_test_2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@VIEW_HBASE_TABLE_TEST_2
+PREHOOK: query: SELECT p.cvalue cvalue
+FROM `VIEW_HBASE_TABLE_TEST_1` `p`
+LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_2` `A1`
+ON `p`.cvalue = `A1`.cvalue
+LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_1` `A2`
+ON `p`.cvalue = `A2`.cvalue
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_test_1
+PREHOOK: Input: default@hbase_table_test_2
+PREHOOK: Input: default@view_hbase_table_test_1
+PREHOOK: Input: default@view_hbase_table_test_2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT p.cvalue cvalue
+FROM `VIEW_HBASE_TABLE_TEST_1` `p`
+LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_2` `A1`
+ON `p`.cvalue = `A1`.cvalue
+LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_1` `A2`
+ON `p`.cvalue = `A2`.cvalue
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_test_1
+POSTHOOK: Input: default@hbase_table_test_2
+POSTHOOK: Input: default@view_hbase_table_test_1
+POSTHOOK: Input: default@view_hbase_table_test_2
+#### A masked pattern was here ####
+PREHOOK: query: DROP VIEW VIEW_HBASE_TABLE_TEST_2
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view_hbase_table_test_2
+PREHOOK: Output: default@view_hbase_table_test_2
+POSTHOOK: query: DROP VIEW VIEW_HBASE_TABLE_TEST_2
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view_hbase_table_test_2
+POSTHOOK: Output: default@view_hbase_table_test_2
+PREHOOK: query: DROP VIEW VIEW_HBASE_TABLE_TEST_1
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view_hbase_table_test_1
+PREHOOK: Output: default@view_hbase_table_test_1
+POSTHOOK: query: DROP VIEW VIEW_HBASE_TABLE_TEST_1
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view_hbase_table_test_1
+POSTHOOK: Output: default@view_hbase_table_test_1
+PREHOOK: query: DROP TABLE HBASE_TABLE_TEST_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_test_2
+PREHOOK: Output: default@hbase_table_test_2
+POSTHOOK: query: DROP TABLE HBASE_TABLE_TEST_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_test_2
+POSTHOOK: Output: default@hbase_table_test_2
+PREHOOK: query: DROP TABLE HBASE_TABLE_TEST_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_test_1
+PREHOOK: Output: default@hbase_table_test_1
+POSTHOOK: query: DROP TABLE HBASE_TABLE_TEST_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_test_1
+POSTHOOK: Output: default@hbase_table_test_1
[7/7] hive git commit: HIVE-14187: JDOPersistenceManager objects
remain cached if MetaStoreClient#close is not called (Mohit Sabharwal,
reviewed by Vaibhav Gumasha, via Sergio Pena)
Posted by sp...@apache.org.
HIVE-14187: JDOPersistenceManager objects remain cached if MetaStoreClient#close is not called (Mohit Sabharwal, reviewed by Vaibhav Gumasha, via Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e045c5a5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e045c5a5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e045c5a5
Branch: refs/heads/branch-2.1
Commit: e045c5a572498f0d84f363ebc6075f0f0fba2e51
Parents: fb99e3e
Author: Mohit Sabharwal <mo...@cloudera.com>
Authored: Fri Jul 15 16:46:04 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:54:42 2016 -0500
----------------------------------------------------------------------
.../hive/metastore/TestHiveMetaStore.java | 51 ++++++++++++++++++++
.../hive/metastore/TestRemoteHiveMetaStore.java | 10 ++--
.../hive/metastore/TestSetUGIOnOnlyClient.java | 4 +-
.../hive/metastore/TestSetUGIOnOnlyServer.java | 4 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 35 ++++++++------
.../hive/metastore/HiveMetaStoreClient.java | 6 +++
6 files changed, 88 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e045c5a5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 9a86fdd..1697780 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import java.lang.reflect.Field;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -33,6 +34,8 @@ import java.util.Set;
import junit.framework.TestCase;
+import org.datanucleus.api.jdo.JDOPersistenceManager;
+import org.datanucleus.api.jdo.JDOPersistenceManagerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -3166,6 +3169,54 @@ public abstract class TestHiveMetaStore extends TestCase {
client.close();
}
+ public void testJDOPersistanceManagerCleanup() throws Exception {
+ if (isThriftClient == false) {
+ return;
+ }
+
+ int numObjectsBeforeClose = getJDOPersistanceManagerCacheSize();
+ HiveMetaStoreClient closingClient = new HiveMetaStoreClient(hiveConf);
+ closingClient.getAllDatabases();
+ closingClient.close();
+ Thread.sleep(5 * 1000); // give HMS time to handle close request
+ int numObjectsAfterClose = getJDOPersistanceManagerCacheSize();
+ Assert.assertTrue(numObjectsBeforeClose == numObjectsAfterClose);
+
+ HiveMetaStoreClient nonClosingClient = new HiveMetaStoreClient(hiveConf);
+ nonClosingClient.getAllDatabases();
+ // Drop connection without calling close. HMS thread deleteContext
+ // will trigger cleanup
+ nonClosingClient.getTTransport().close();
+ Thread.sleep(5 * 1000);
+ int numObjectsAfterDroppedConnection = getJDOPersistanceManagerCacheSize();
+ Assert.assertTrue(numObjectsAfterClose == numObjectsAfterDroppedConnection);
+ }
+
+ private static int getJDOPersistanceManagerCacheSize() {
+ JDOPersistenceManagerFactory jdoPmf;
+ Set<JDOPersistenceManager> pmCacheObj;
+ Field pmCache;
+ Field pmf;
+ try {
+ pmf = ObjectStore.class.getDeclaredField("pmf");
+ if (pmf != null) {
+ pmf.setAccessible(true);
+ jdoPmf = (JDOPersistenceManagerFactory) pmf.get(null);
+ pmCache = JDOPersistenceManagerFactory.class.getDeclaredField("pmCache");
+ if (pmCache != null) {
+ pmCache.setAccessible(true);
+ pmCacheObj = (Set<JDOPersistenceManager>) pmCache.get(jdoPmf);
+ if (pmCacheObj != null) {
+ return pmCacheObj.size();
+ }
+ }
+ }
+ } catch (Exception ex) {
+ System.out.println(ex);
+ }
+ return -1;
+ }
+
private HiveMetaHookLoader getHookLoader() {
HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e045c5a5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
index 491d093..faea0e4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.shims.ShimLoader;
public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
private static boolean isServerStarted = false;
+ private static int port;
public TestRemoteHiveMetaStore() {
super();
@@ -37,21 +38,22 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
if (isServerStarted) {
assertNotNull("Unable to connect to the MetaStore server", client);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
return;
}
- int port = MetaStoreUtils.findFreePort();
+ port = MetaStoreUtils.findFreePort();
System.out.println("Starting MetaStore Server on port " + port);
MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
isServerStarted = true;
// This is default case with setugi off for both client and server
- createClient(false, port);
+ createClient(false);
}
- protected void createClient(boolean setugi, int port) throws Exception {
+ protected void createClient(boolean setugi) throws Exception {
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI,setugi);
client = new HiveMetaStoreClient(hiveConf);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/e045c5a5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java
index 2c6d567..29768c1 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.hive.metastore;
public class TestSetUGIOnOnlyClient extends TestRemoteHiveMetaStore{
@Override
- protected void createClient(boolean setugi, int port) throws Exception {
+ protected void createClient(boolean setugi) throws Exception {
// turn it on for client.
- super.createClient(true, port);
+ super.createClient(true);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e045c5a5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java
index 6c3fbf6..4a46f75 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.hive.metastore;
public class TestSetUGIOnOnlyServer extends TestSetUGIOnBothClientServer {
@Override
- protected void createClient(boolean setugi, int port) throws Exception {
+ protected void createClient(boolean setugi) throws Exception {
// It is turned on for both client and server because of super class. Turn it off for client.
- super.createClient(false, port);
+ super.createClient(false);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e045c5a5/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 66bf23c..e46e4d1a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -281,7 +281,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
};
- private final void logAuditEvent(String cmd) {
+ private static final void logAuditEvent(String cmd) {
if (cmd == null) {
return;
}
@@ -304,7 +304,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
address, cmd).toString());
}
- String getIPAddress() {
+ private static String getIPAddress() {
if (useSasl) {
if (saslServer != null && saslServer.getRemoteAddress() != null) {
return saslServer.getRemoteAddress().getHostAddress();
@@ -747,7 +747,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
- private void logInfo(String m) {
+ private static void logInfo(String m) {
LOG.info(threadLocalId.get().toString() + ": " + m);
logAuditEvent(m);
}
@@ -820,17 +820,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@Override
public void shutdown() {
- logInfo("Metastore shutdown started...");
- RawStore ms = threadLocalMS.get();
- if (ms != null) {
- try {
- ms.shutdown();
- } finally {
- threadLocalConf.remove();
- threadLocalMS.remove();
- }
- }
- logInfo("Metastore shutdown complete.");
+ cleanupRawStore();
}
@Override
@@ -6786,6 +6776,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
} catch (Exception e) {
LOG.warn("Error Reporting Metastore close connection to Metrics system", e);
}
+ // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already
+ // cleaned up thread local RawStore. Otherwise, do it now.
+ cleanupRawStore();
}
@Override
@@ -6813,6 +6806,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
+ private static void cleanupRawStore() {
+ RawStore rs = HMSHandler.getRawStore();
+ if (rs != null) {
+ HMSHandler.logInfo("Cleaning up thread local RawStore...");
+ try {
+ rs.shutdown();
+ } finally {
+ HMSHandler.threadLocalConf.remove();
+ HMSHandler.removeRawStore();
+ }
+ HMSHandler.logInfo("Done cleaning up thread local RawStore");
+ }
+ }
+
private static void signalOtherThreadsToStart(final TServer server, final Lock startLock,
final Condition startCondition,
final AtomicBoolean startedServing) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e045c5a5/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 0b2f77b..129512d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
@@ -324,6 +325,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
metastoreUris[index] = tmp;
}
+ @VisibleForTesting
+ public TTransport getTTransport() {
+ return transport;
+ }
+
@Override
public boolean isLocalMetaStore() {
return localMetaStore;
[4/7] hive git commit: HIVE-14519 : Multi insert query bug (Ashutosh
Chauhan via Yongzhi Chen)
Posted by sp...@apache.org.
HIVE-14519 : Multi insert query bug (Ashutosh Chauhan via Yongzhi Chen)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f23f9f2e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f23f9f2e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f23f9f2e
Branch: refs/heads/branch-2.1
Commit: f23f9f2e05facc26de488a5cedc82f7ec8d1d27d
Parents: 23c98ad
Author: Yongzhi Chen <yc...@apache.org>
Authored: Sat Aug 13 12:48:51 2016 -0700
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:46:24 2016 -0500
----------------------------------------------------------------------
.../optimizer/physical/NullScanOptimizer.java | 12 +++++++
.../queries/clientpositive/inputwherefalse.q | 15 +++++++++
.../clientpositive/inputwherefalse.q.out | 33 ++++++++++++++++++++
3 files changed, 60 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f23f9f2e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java
index 200e098..49b839a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java
@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.LimitOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -90,6 +91,17 @@ public class NullScanOptimizer implements PhysicalPlanResolver {
return null;
}
+ int numOfndPeers = 0;
+ if (filter.getParentOperators() != null) {
+ for (Operator<?> fParent : filter.getParentOperators()) {
+ if (fParent.getChildOperators() != null) {
+ numOfndPeers += fParent.getChildOperators().size();
+ if (numOfndPeers > 1)
+ return null;
+ }
+ }
+ }
+
WalkerCtx ctx = (WalkerCtx) procCtx;
for (Node op : stack) {
if (op instanceof TableScanOperator) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f23f9f2e/ql/src/test/queries/clientpositive/inputwherefalse.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/inputwherefalse.q b/ql/src/test/queries/clientpositive/inputwherefalse.q
new file mode 100644
index 0000000..c9186e6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/inputwherefalse.q
@@ -0,0 +1,15 @@
+From (select * from src) a
+insert overwrite directory 'target/warehouse/destemp.out/dir1/'
+select key
+where key=200 limit 1
+insert overwrite directory 'target/warehouse/destemp.out/dir2/'
+select 'header'
+where 1=2
+insert overwrite directory 'target/warehouse/destemp.out/dir3/'
+select key
+where key = 100 limit 1;
+
+dfs -cat ${system:test.warehouse.dir}/destemp.out/dir1/000000_0;
+dfs -cat ${system:test.warehouse.dir}/destemp.out/dir2/000000_0;
+dfs -cat ${system:test.warehouse.dir}/destemp.out/dir3/000000_0;
+dfs -rmr ${system:test.warehouse.dir}/destemp.out;
http://git-wip-us.apache.org/repos/asf/hive/blob/f23f9f2e/ql/src/test/results/clientpositive/inputwherefalse.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/inputwherefalse.q.out b/ql/src/test/results/clientpositive/inputwherefalse.q.out
new file mode 100644
index 0000000..ecb7300
--- /dev/null
+++ b/ql/src/test/results/clientpositive/inputwherefalse.q.out
@@ -0,0 +1,33 @@
+PREHOOK: query: From (select * from src) a
+insert overwrite directory 'target/warehouse/destemp.out/dir1/'
+select key
+where key=200 limit 1
+insert overwrite directory 'target/warehouse/destemp.out/dir2/'
+select 'header'
+where 1=2
+insert overwrite directory 'target/warehouse/destemp.out/dir3/'
+select key
+where key = 100 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: target/warehouse/destemp.out/dir1
+PREHOOK: Output: target/warehouse/destemp.out/dir2
+PREHOOK: Output: target/warehouse/destemp.out/dir3
+POSTHOOK: query: From (select * from src) a
+insert overwrite directory 'target/warehouse/destemp.out/dir1/'
+select key
+where key=200 limit 1
+insert overwrite directory 'target/warehouse/destemp.out/dir2/'
+select 'header'
+where 1=2
+insert overwrite directory 'target/warehouse/destemp.out/dir3/'
+select key
+where key = 100 limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: target/warehouse/destemp.out/dir1
+POSTHOOK: Output: target/warehouse/destemp.out/dir2
+POSTHOOK: Output: target/warehouse/destemp.out/dir3
+200
+100
+#### A masked pattern was here ####
[3/7] hive git commit: HIVE-13749: Memory leak in Hive Metastore
(Naveen Gangam, reviewed by Thejas M Nair, Aihua Xu)
Posted by sp...@apache.org.
HIVE-13749: Memory leak in Hive Metastore (Naveen Gangam, reviewed by Thejas M Nair, Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/23c98ad6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/23c98ad6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/23c98ad6
Branch: refs/heads/branch-2.1
Commit: 23c98ad6a5cb62180f7ef6649069632cd39fbc24
Parents: 72b039c
Author: Aihua Xu <ai...@apache.org>
Authored: Fri Jul 8 09:54:12 2016 -0400
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:46:09 2016 -0500
----------------------------------------------------------------------
.../src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/23c98ad6/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index c7e3a6a..66bf23c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -826,6 +826,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
try {
ms.shutdown();
} finally {
+ threadLocalConf.remove();
threadLocalMS.remove();
}
}
[5/7] hive git commit: HIVE-14715: Hive throws NumberFormatException
with query with Null value (Yongzhi Chen, reviewed by Aihua Xu)
Posted by sp...@apache.org.
HIVE-14715: Hive throws NumberFormatException with query with Null value (Yongzhi Chen, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7db2ec5d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7db2ec5d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7db2ec5d
Branch: refs/heads/branch-2.1
Commit: 7db2ec5d8da101b59d8586031321327e06f52870
Parents: f23f9f2
Author: Yongzhi Chen <yc...@apache.org>
Authored: Wed Sep 7 21:30:01 2016 -0400
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:53:52 2016 -0500
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +-
.../queries/clientpositive/groupby_nullvalues.q | 29 +++++++
.../clientpositive/groupby_nullvalues.q.out | 83 ++++++++++++++++++++
3 files changed, 114 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7db2ec5d/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index ede0b6e..8d900b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -5157,6 +5157,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
colExprMap);
int keyLength = reduceKeys.size();
+ int numOfColsRmedFromkey = grpByExprs.size() - keyLength;
// add a key for reduce sink
if (groupingSetsPresent) {
@@ -5186,7 +5187,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
reduceSinkOutputRowResolver, outputValueColumnNames, reduceValues, colExprMap);
} else {
// Put partial aggregation results in reduceValues
- int inputField = reduceKeys.size();
+ int inputField = reduceKeys.size() + numOfColsRmedFromkey;
for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/7db2ec5d/ql/src/test/queries/clientpositive/groupby_nullvalues.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_nullvalues.q b/ql/src/test/queries/clientpositive/groupby_nullvalues.q
new file mode 100644
index 0000000..85ccb38
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/groupby_nullvalues.q
@@ -0,0 +1,29 @@
+set hive.cbo.enable=false;
+drop table if exists paqtest;
+CREATE TABLE paqtest(
+c1 int,
+s1 string,
+s2 string,
+bn1 bigint)
+ROW FORMAT SERDE
+'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';
+
+insert into paqtest values (58, '', 'ABC', 0);
+
+SELECT
+'PM' AS cy,
+c1,
+NULL AS iused,
+NULL AS itp,
+s2,
+NULL AS cvg,
+NULL AS acavg,
+sum(bn1) AS cca
+FROM paqtest
+WHERE (s1 IS NULL OR length(s1) = 0)
+GROUP BY 'Pricing mismatch', c1, NULL, NULL, s2, NULL, NULL;
+drop table paqtest;
http://git-wip-us.apache.org/repos/asf/hive/blob/7db2ec5d/ql/src/test/results/clientpositive/groupby_nullvalues.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_nullvalues.q.out b/ql/src/test/results/clientpositive/groupby_nullvalues.q.out
new file mode 100644
index 0000000..aae2a12
--- /dev/null
+++ b/ql/src/test/results/clientpositive/groupby_nullvalues.q.out
@@ -0,0 +1,83 @@
+PREHOOK: query: drop table if exists paqtest
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists paqtest
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE paqtest(
+c1 int,
+s1 string,
+s2 string,
+bn1 bigint)
+ROW FORMAT SERDE
+'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@paqtest
+POSTHOOK: query: CREATE TABLE paqtest(
+c1 int,
+s1 string,
+s2 string,
+bn1 bigint)
+ROW FORMAT SERDE
+'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@paqtest
+PREHOOK: query: insert into paqtest values (58, '', 'ABC', 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@paqtest
+POSTHOOK: query: insert into paqtest values (58, '', 'ABC', 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@paqtest
+POSTHOOK: Lineage: paqtest.bn1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
+POSTHOOK: Lineage: paqtest.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: paqtest.s1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: paqtest.s2 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: SELECT
+'PM' AS cy,
+c1,
+NULL AS iused,
+NULL AS itp,
+s2,
+NULL AS cvg,
+NULL AS acavg,
+sum(bn1) AS cca
+FROM paqtest
+WHERE (s1 IS NULL OR length(s1) = 0)
+GROUP BY 'Pricing mismatch', c1, NULL, NULL, s2, NULL, NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@paqtest
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+'PM' AS cy,
+c1,
+NULL AS iused,
+NULL AS itp,
+s2,
+NULL AS cvg,
+NULL AS acavg,
+sum(bn1) AS cca
+FROM paqtest
+WHERE (s1 IS NULL OR length(s1) = 0)
+GROUP BY 'Pricing mismatch', c1, NULL, NULL, s2, NULL, NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@paqtest
+#### A masked pattern was here ####
+PM 58 NULL NULL ABC NULL NULL 0
+PREHOOK: query: drop table paqtest
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@paqtest
+PREHOOK: Output: default@paqtest
+POSTHOOK: query: drop table paqtest
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@paqtest
+POSTHOOK: Output: default@paqtest
[2/7] hive git commit: HIVE-14215: Displaying inconsistent CPU usage
data with MR execution engine (Peter Vary,
reviewed by Aihua Xu and Sergio Pena)
Posted by sp...@apache.org.
HIVE-14215: Displaying inconsistent CPU usage data with MR execution engine (Peter Vary, reviewed by Aihua Xu and Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/72b039c8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/72b039c8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/72b039c8
Branch: refs/heads/branch-2.1
Commit: 72b039c82ffc033ae8a1f9ae79f6611467aacfc5
Parents: 3701a7a
Author: Peter Vary <pv...@cloudera.com>
Authored: Thu Jul 14 11:27:18 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 15:45:52 2016 -0500
----------------------------------------------------------------------
.../hive/ql/exec/mr/HadoopJobExecHelper.java | 25 ++++++++++----------
1 file changed, 13 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/72b039c8/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
index 5656f9a..a894bed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
@@ -377,6 +377,19 @@ public class HadoopJobExecHelper {
reportTime = System.currentTimeMillis();
}
+ Counters ctrs = th.getCounters();
+
+ if (ctrs != null) {
+ Counter counterCpuMsec = ctrs.findCounter("org.apache.hadoop.mapred.Task$Counter",
+ "CPU_MILLISECONDS");
+ if (counterCpuMsec != null) {
+ long newCpuMSec = counterCpuMsec.getValue();
+ if (newCpuMSec > cpuMsec) {
+ cpuMsec = newCpuMSec;
+ }
+ }
+ }
+
if (cpuMsec > 0) {
String status = "MapReduce Total cumulative CPU time: " + Utilities.formatMsecToStr(cpuMsec);
console.printInfo(status);
@@ -385,7 +398,6 @@ public class HadoopJobExecHelper {
boolean success;
- Counters ctrs = th.getCounters();
if (fatal) {
success = false;
} else {
@@ -403,17 +415,6 @@ public class HadoopJobExecHelper {
}
}
- if (ctrs != null) {
- Counter counterCpuMsec = ctrs.findCounter("org.apache.hadoop.mapred.Task$Counter",
- "CPU_MILLISECONDS");
- if (counterCpuMsec != null) {
- long newCpuMSec = counterCpuMsec.getValue();
- if (newCpuMSec > cpuMsec) {
- cpuMsec = newCpuMSec;
- }
- }
- }
-
MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString());
mapRedStats.setCounters(ctrs);