You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ct...@apache.org on 2017/04/07 12:49:04 UTC
[1/3] hive git commit: HIVE-15538: Test HIVE-13884 with more complex
query predicates (Marta Kuczora via Chaoyu Tang)
Repository: hive
Updated Branches:
refs/heads/master a01a6a348 -> 0d0e4976c
HIVE-15538: Test HIVE-13884 with more complex query predicates (Marta Kuczora via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5ba09e88
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5ba09e88
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5ba09e88
Branch: refs/heads/master
Commit: 5ba09e883ea1d26d3306326dabb01aff10c71e8d
Parents: a01a6a3
Author: Chaoyu Tang <ct...@cloudera.com>
Authored: Fri Apr 7 08:39:07 2017 -0400
Committer: Chaoyu Tang <ct...@cloudera.com>
Committed: Fri Apr 7 08:39:07 2017 -0400
----------------------------------------------------------------------
.../ql/TestMetaStoreLimitPartitionRequest.java | 319 +++++++++++++++++++
.../hadoop/hive/metastore/HiveMetaStore.java | 6 +-
2 files changed, 323 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5ba09e88/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
new file mode 100644
index 0000000..191d4a3
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * This unit test is for testing HIVE-13884 with more complex queries and
+ * hive.metastore.limit.partition.request enabled.
+ * It covers cases when the query predicates can be pushed down and the
+ * number of partitions can be retrieved via directSQL.
+ * It also covers cases when the number of partitions cannot be retrieved
+ * via directSQL, so it falls back to ORM.
+ */
+public class TestMetaStoreLimitPartitionRequest {
+
+ private static final String DB_NAME = "max_partition_test_db";
+ private static final String TABLE_NAME = "max_partition_test_table";
+ private static int PARTITION_REQUEST_LIMIT = 4;
+ private static MiniHS2 miniHS2 = null;
+ private static HiveConf conf;
+ private Connection hs2Conn = null;
+ private Statement stmt;
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ Class.forName(MiniHS2.getJdbcDriverName());
+ conf = new HiveConf();
+ DriverManager.setLoginTimeout(0);
+
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ conf.setIntVar(HiveConf.ConfVars.METASTORE_LIMIT_PARTITION_REQUEST, PARTITION_REQUEST_LIMIT);
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_INTEGER_JDO_PUSHDOWN, true);
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, true);
+ conf.setBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING, true);
+ conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, false);
+
+ miniHS2 = new MiniHS2.Builder().withConf(conf).build();
+ Map<String, String> overlayProps = new HashMap<String, String>();
+ miniHS2.start(overlayProps);
+ createDb();
+ }
+
+ private static void createDb() throws Exception {
+ Connection conn =
+ DriverManager.getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+ Statement stmt2 = conn.createStatement();
+ stmt2.execute("DROP DATABASE IF EXISTS " + DB_NAME + " CASCADE");
+ stmt2.execute("CREATE DATABASE " + DB_NAME);
+ stmt2.close();
+ conn.close();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(DB_NAME),
+ System.getProperty("user.name"), "bar");
+ stmt = hs2Conn.createStatement();
+ stmt.execute("USE " + DB_NAME);
+ createTable();
+ }
+
+ private void createTable() throws Exception {
+ String tmpTableName = TABLE_NAME + "_tmp";
+ stmt.execute("CREATE TABLE " + tmpTableName
+ + " (id string, value string, num string, ds date) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' STORED AS TEXTFILE");
+ stmt.execute("INSERT OVERWRITE TABLE " + tmpTableName
+ + " VALUES ('1', 'value1', '25', '2008-04-09'), ('2', 'value2', '30', '2008-04-09'), "
+ + "('3', 'value3', '35', '2008-04-09'), ('4', 'value4', '40', '2008-04-09'), "
+ + "('5', 'value5', '25', '2008-05-09'), ('6', 'value6', '30', '2008-05-09'), "
+ + "('7', 'value7', '35', '2008-05-09'), ('8', 'value8', '40', '2008-05-09'), "
+ + "('9', 'value9', '25', '2009-04-09'), ('10', 'value10', '30', '2009-04-09'), "
+ + "('11', 'value11', '35', '2009-04-09'), ('12', 'value12', '40', '2009-04-09')");
+
+ stmt.execute("CREATE TABLE " + TABLE_NAME + " (id string, value string) PARTITIONED BY (num string, ds date)");
+ stmt.execute("INSERT OVERWRITE TABLE " + TABLE_NAME + " PARTITION (num, ds) SELECT id, value, num, ds FROM " + tmpTableName);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ String tmpTableName = TABLE_NAME + "_tmp";
+ stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME);
+ stmt.execute("DROP TABLE IF EXISTS " + tmpTableName);
+ stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME + "_num_tmp");
+
+ if (hs2Conn != null) {
+ hs2Conn.close();
+ }
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ if (miniHS2 != null && miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ }
+
+ /* Tests with queries which can be pushed down and executed with directSQL */
+
+ @Test
+ public void testSimpleQueryWithDirectSql() throws Exception {
+ String queryString = "select value from %s where num='25' and ds='2008-04-09'";
+ executeQuery(queryString, "value1");
+ }
+
+ @Test
+ public void testMoreComplexQueryWithDirectSql() throws Exception {
+ String queryString = "select value from %s where (ds between '2009-01-01' and '2009-12-31' and num='25') or (ds between '2008-01-01' and '2008-12-31' and num='30')";
+ executeQuery(queryString, "value2", "value6", "value9");
+ }
+
+ /*
+ * Tests with queries which can be pushed down and executed with directSQL, but the number of
+ * partitions which should be fetched is bigger than the maximum set by the
+ * hive.metastore.limit.partition.request parameter.
+ */
+
+ @Test
+ public void testSimpleQueryWithDirectSqlTooManyPartitions() throws Exception {
+ String queryString = "select value from %s where ds>'2008-04-20'";
+ executeQueryExceedPartitionLimit(queryString, 8);
+ }
+
+ @Test
+ public void testMoreComplexQueryWithDirectSqlTooManyPartitions() throws Exception {
+ String queryString = "select value from %s where num='25' or (num='30' and ds between '2008-01-01' and '2008-12-31')";
+ executeQueryExceedPartitionLimit(queryString, 5);
+ }
+
+ /*
+ * Tests with queries which cannot be executed with directSQL, because of type mismatch. The type
+ * of the num column is string, but the parameters used in the where clause are numbers. After
+ * falling back to ORM, the number of partitions can be fetched by the
+ * ObjectStore.getNumPartitionsViaOrmFilter method.
+ */
+
+ @Test
+ public void testQueryWithFallbackToORM1() throws Exception {
+ String queryString = "select value from %s where num!=25 and num!=35 and num!=40";
+ executeQuery(queryString, "value2", "value6", "value10");
+ }
+
+ @Test
+ public void testQueryWithFallbackToORMTooManyPartitions1() throws Exception {
+ String queryString = "select value from %s where num=30 or num=25";
+ executeQueryExceedPartitionLimit(queryString, 6);
+ }
+
+ /*
+ * Tests with queries which cannot be executed with directSQL, because of type mismatch. The type
+ * of the num column is string, but the parameters used in the where clause are numbers. After
+ * falling back to ORM the number of partitions cannot be fetched by the
+ * ObjectStore.getNumPartitionsViaOrmFilter method. They are fetched by the
+ * ObjectStore.getPartitionNamesPrunedByExprNoTxn method.
+ */
+
+ @Test
+ public void testQueryWithFallbackToORM2() throws Exception {
+ String queryString = "select value from %s where num!=25 and ds='2008-04-09'";
+ executeQuery(queryString, "value2", "value3", "value4");
+ }
+
+ @Test
+ public void testQueryWithFallbackToORM3() throws Exception {
+ String queryString = "select value from %s where num between 26 and 31";
+ executeQuery(queryString, "value2", "value6", "value10");
+ }
+
+ @Test
+ public void testQueryWithFallbackToORMTooManyPartitions2() throws Exception {
+ String queryString = "select value from %s where num!=25 and (ds='2008-04-09' or ds='2008-05-09')";
+ executeQueryExceedPartitionLimit(queryString, 6);
+ }
+
+ @Test
+ public void testQueryWithFallbackToORMTooManyPartitions3() throws Exception {
+ String queryString = "select value from %s where num>=30";
+ executeQueryExceedPartitionLimit(queryString, 9);
+ }
+
+ @Test
+ public void testQueryWithFallbackToORMTooManyPartitions4() throws Exception {
+ String queryString = "select value from %s where num between 20 and 50";
+ executeQueryExceedPartitionLimit(queryString, 12);
+ }
+
+ /*
+ * Tests with queries which cannot be executed with directSQL, because the contain like or in.
+ * After falling back to ORM the number of partitions cannot be fetched by the
+ * ObjectStore.getNumPartitionsViaOrmFilter method. They are fetched by the
+ * ObjectStore.getPartitionNamesPrunedByExprNoTxn method.
+ */
+
+ @Test
+ public void testQueryWithInWithFallbackToORM() throws Exception {
+ setupNumTmpTable();
+ String queryString = "select value from %s a where ds='2008-04-09' and a.num in (select value from " + TABLE_NAME + "_num_tmp)";
+ executeQuery(queryString, "value1", "value2");
+ }
+
+ @Test
+ public void testQueryWithInWithFallbackToORMTooManyPartitions() throws Exception {
+ setupNumTmpTable();
+ String queryString = "select value from %s a where a.num in (select value from " + TABLE_NAME + "_num_tmp)";
+ executeQueryExceedPartitionLimit(queryString, 12);
+ }
+
+ @Test
+ public void testQueryWithInWithFallbackToORMTooManyPartitions2() throws Exception {
+ setupNumTmpTable();
+ String queryString = "select value from %s a where a.num in (select value from " + TABLE_NAME + "_num_tmp where value='25')";
+ executeQueryExceedPartitionLimit(queryString, 12);
+ }
+
+ @Test
+ public void testQueryWithLikeWithFallbackToORMTooManyPartitions() throws Exception {
+ String queryString = "select value from %s where num like '3%%'";
+ executeQueryExceedPartitionLimit(queryString, 6);
+ }
+
+ private void setupNumTmpTable() throws SQLException {
+ stmt.execute("CREATE TABLE " + TABLE_NAME + "_num_tmp (value string)");
+ stmt.execute("INSERT INTO " + TABLE_NAME + "_num_tmp VALUES ('25')");
+ stmt.execute("INSERT INTO " + TABLE_NAME + "_num_tmp VALUES ('30')");
+ }
+
+ private void executeQuery(String query, String... expectedValues) throws SQLException {
+ String queryStr = String.format(query, TABLE_NAME);
+ ResultSet result = stmt.executeQuery(queryStr);
+ assertTrue(result != null);
+ Set<String> expectedValueSet = new HashSet<>(Arrays.asList(expectedValues));
+ Set<String> resultValues = getResultValues(result);
+ String errorMsg = getWrongResultErrorMsg(queryStr, expectedValueSet.toString(), resultValues.toString());
+ assertTrue(errorMsg, resultValues.equals(expectedValueSet));
+ }
+
+ private Set<String> getResultValues(ResultSet result) throws SQLException {
+ Set<String> resultValues = new HashSet<>();
+ while(result.next()) {
+ resultValues.add(result.getString(1));
+ }
+ return resultValues;
+ }
+
+ private void executeQueryExceedPartitionLimit(String query, int expectedPartitionNumber) throws Exception {
+ try {
+ String queryStr = String.format(query, TABLE_NAME);
+ stmt.executeQuery(queryStr);
+ fail("The query should have failed, because the number of requested partitions are bigger than "
+ + PARTITION_REQUEST_LIMIT);
+ } catch (HiveSQLException e) {
+ String exceedLimitMsg = String.format(HiveMetaStore.PARTITION_NUMBER_EXCEED_LIMIT_MSG, expectedPartitionNumber,
+ TABLE_NAME, PARTITION_REQUEST_LIMIT, ConfVars.METASTORE_LIMIT_PARTITION_REQUEST.varname);
+ assertTrue(getWrongExceptionMessage(exceedLimitMsg, e.getMessage()),
+ e.getMessage().contains(exceedLimitMsg.toString()));
+ }
+ }
+
+ private String getWrongResultErrorMsg(String query, String expectedValues, String resultValues) {
+ StringBuilder errorMsg = new StringBuilder();
+ errorMsg.append("The query '");
+ errorMsg.append(query);
+ errorMsg.append("' returned wrong values. It returned the values ");
+ errorMsg.append(resultValues);
+ errorMsg.append(" instead of the expected ");
+ errorMsg.append(expectedValues);
+ return errorMsg.toString();
+ }
+
+ private String getWrongExceptionMessage(String exceedLimitMsg, String exceptionMessage) {
+ StringBuilder errorMsg = new StringBuilder();
+ errorMsg.append("The message of the exception doesn't contain the expected '");
+ errorMsg.append(exceedLimitMsg.toString());
+ errorMsg.append("'. It is: ");
+ errorMsg.append(exceptionMessage);
+ return errorMsg.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5ba09e88/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 3aabe22..8906214 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -162,6 +162,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*/
public class HiveMetaStore extends ThriftHiveMetastore {
public static final Logger LOG = LoggerFactory.getLogger(HiveMetaStore.class);
+ public static final String PARTITION_NUMBER_EXCEED_LIMIT_MSG =
+ "Number of partitions scanned (=%d) on table '%s' exceeds limit (=%d). This is controlled on the metastore server by %s.";
// boolean that tells if the HiveMetaStore (remote) server is being used.
// Can be used to determine if the calls to metastore api (HMSHandler) are being made with
@@ -3528,8 +3530,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
int partitionRequest = (maxToFetch < 0) ? numPartitions : maxToFetch;
if (partitionRequest > partitionLimit) {
String configName = ConfVars.METASTORE_LIMIT_PARTITION_REQUEST.varname;
- throw new MetaException(String.format("Number of partitions scanned (=%d) on table '%s' exceeds limit" +
- " (=%d). This is controlled on the metastore server by %s.", partitionRequest, tblName, partitionLimit, configName));
+ throw new MetaException(String.format(PARTITION_NUMBER_EXCEED_LIMIT_MSG, partitionRequest,
+ tblName, partitionLimit, configName));
}
}
}
[2/3] hive git commit: HIVE-16394: HoS does not support queue name
change in middle of session (Chaoyu Tang, reviewed by Xuefu Zhang, Rui Li)
Posted by ct...@apache.org.
HIVE-16394: HoS does not support queue name change in middle of session (Chaoyu Tang, reviewed by Xuefu Zhang, Rui Li)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/72fb816b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/72fb816b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/72fb816b
Branch: refs/heads/master
Commit: 72fb816b814ee931d8a2c93e1dddcb9478f3674a
Parents: 5ba09e8
Author: Chaoyu Tang <ct...@cloudera.com>
Authored: Fri Apr 7 08:41:46 2017 -0400
Committer: Chaoyu Tang <ct...@cloudera.com>
Committed: Fri Apr 7 08:41:46 2017 -0400
----------------------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/72fb816b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7d4a6a0..7e11649 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3689,6 +3689,9 @@ public class HiveConf extends Configuration {
}
} else if (name.startsWith("hive.spark")) { // Remote Spark Context property.
result = true;
+ } else if (name.equals("mapreduce.job.queuename")) {
+ // a special property starting with mapreduce that we would also like to effect if it changes
+ result = true;
}
return result;
[3/3] hive git commit: HIVE-16334: Query lock contains the query
string, which can cause OOM on ZooKeeper (Peter Vary via Chaoyu Tang)
Posted by ct...@apache.org.
HIVE-16334: Query lock contains the query string, which can cause OOM on ZooKeeper (Peter Vary via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d0e4976
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d0e4976
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d0e4976
Branch: refs/heads/master
Commit: 0d0e4976c155a450b93a14ce678b6ffb21f639df
Parents: 72fb816
Author: Chaoyu Tang <ct...@cloudera.com>
Authored: Fri Apr 7 08:44:28 2017 -0400
Committer: Chaoyu Tang <ct...@cloudera.com>
Committed: Fri Apr 7 08:44:28 2017 -0400
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 ++
.../hadoop/hive/ql/lockmgr/DummyTxnManager.java | 3 +-
.../hadoop/hive/ql/lockmgr/HiveLockObject.java | 18 ++++++++----
.../hive/ql/lockmgr/HiveTxnManagerImpl.java | 5 ++--
.../hive/ql/lockmgr/TestDummyTxnManager.java | 4 +--
.../ql/lockmgr/TestEmbeddedLockManager.java | 4 ++-
.../hive/ql/lockmgr/TestHiveLockObject.java | 30 ++++++++++++++++++--
.../zookeeper/TestZookeeperLockManager.java | 2 +-
8 files changed, 55 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7e11649..4f7f1e7 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1771,6 +1771,9 @@ public class HiveConf extends Configuration {
HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", false,
"This param is to control whether or not only do lock on queries\n" +
"that need to execute at least one mapred job."),
+ HIVE_LOCK_QUERY_STRING_MAX_LENGTH("hive.lock.query.string.max.length", 1000000,
+ "The maximum length of the query string to store in the lock.\n" +
+ "The default value is 1000000, since the data limit of a znode is 1MB"),
// Zookeeper related configs
HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "",
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
index 53ee9c8..24df25b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
@@ -301,7 +301,8 @@ class DummyTxnManager extends HiveTxnManagerImpl {
new HiveLockObject.HiveLockObjectData(plan.getQueryId(),
String.valueOf(System.currentTimeMillis()),
"IMPLICIT",
- plan.getQueryStr());
+ plan.getQueryStr(),
+ conf);
if (db != null) {
locks.add(new HiveLockObj(new HiveLockObject(db.getName(), lockData),
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index fff03df..a514339 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hive.common.StringInternUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -48,16 +49,23 @@ public class HiveLockObject {
* Note: The parameters are used to uniquely identify a HiveLockObject.
* The parameters will be stripped off any ':' characters in order not
* to interfere with the way the data is serialized (':' delimited string).
+ * The query string might be truncated depending on HIVE_LOCK_QUERY_STRING_MAX_LENGTH
+ * @param queryId The query identifier will be added to the object without change
+ * @param lockTime The lock time will be added to the object without change
+ * @param lockMode The lock mode will be added to the object without change
+ * @param queryStr The query string might be truncated based on
+ * HIVE_LOCK_QUERY_STRING_MAX_LENGTH conf variable
+ * @param conf The hive configuration based on which we decide if we should truncate the query
+ * string or not
*/
- public HiveLockObjectData(String queryId,
- String lockTime,
- String lockMode,
- String queryStr) {
+ public HiveLockObjectData(String queryId, String lockTime, String lockMode, String queryStr,
+ HiveConf conf) {
this.queryId = removeDelimiter(queryId);
this.lockTime = StringInternUtils.internIfNotNull(removeDelimiter(lockTime));
this.lockMode = removeDelimiter(lockMode);
this.queryStr = StringInternUtils.internIfNotNull(
- removeDelimiter(queryStr == null ? null : queryStr.trim()));
+ queryStr == null ? null : StringUtils.substring(removeDelimiter(queryStr.trim()), 0,
+ conf.getIntVar(HiveConf.ConfVars.HIVE_LOCK_QUERY_STRING_MAX_LENGTH)));
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
index a371a5a..9fa416c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
@@ -93,7 +93,8 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
new HiveLockObjectData(lockTbl.getQueryId(),
String.valueOf(System.currentTimeMillis()),
"EXPLICIT",
- lockTbl.getQueryStr());
+ lockTbl.getQueryStr(),
+ conf);
if (partSpec == null) {
HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
@@ -151,7 +152,7 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
HiveLockObjectData lockData =
new HiveLockObjectData(lockDb.getQueryId(),
String.valueOf(System.currentTimeMillis()),
- "EXPLICIT", lockDb.getQueryStr());
+ "EXPLICIT", lockDb.getQueryStr(), conf);
HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
if (lck == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
index de3b8ad..9d27d21 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
@@ -147,9 +147,9 @@ public class TestDummyTxnManager {
String path1 = "path1";
String path2 = "path2";
HiveLockObjectData lockData1 = new HiveLockObjectData(
- "query1", "1", "IMPLICIT", "drop table table1");
+ "query1", "1", "IMPLICIT", "drop table table1", conf);
HiveLockObjectData lockData2 = new HiveLockObjectData(
- "query1", "1", "IMPLICIT", "drop table table1");
+ "query1", "1", "IMPLICIT", "drop table table1", conf);
// Start with the following locks:
// [path1, shared]
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestEmbeddedLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestEmbeddedLockManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestEmbeddedLockManager.java
index 0afbc1c..95a9856 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestEmbeddedLockManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestEmbeddedLockManager.java
@@ -27,6 +27,7 @@ import org.junit.Assert;
public class TestEmbeddedLockManager extends TestCase {
private int counter;
+ private HiveConf conf = new HiveConf();
public void testLocking() throws LockException {
HiveConf conf = new HiveConf();
@@ -119,7 +120,8 @@ public class TestEmbeddedLockManager extends TestCase {
}
private HiveLockObject lockObj(String path, String query) {
- HiveLockObjectData data = new HiveLockObjectData(String.valueOf(++counter), null, null, query);
+ HiveLockObjectData data = new HiveLockObjectData(String.valueOf(++counter), null, null,
+ query, conf);
return new HiveLockObject(path.split("/"), data);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestHiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestHiveLockObject.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestHiveLockObject.java
index 19cb129..024114f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestHiveLockObject.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestHiveLockObject.java
@@ -20,15 +20,20 @@ package org.apache.hadoop.hive.ql.lockmgr;
import junit.framework.Assert;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
import org.junit.Test;
public class TestHiveLockObject {
+ private HiveConf conf = new HiveConf();
+
@Test
public void testEqualsAndHashCode() {
- HiveLockObjectData data1 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01", "select * from mytable");
- HiveLockObjectData data2 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01", "select * from mytable");
+ HiveLockObjectData data1 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ "select * from mytable", conf);
+ HiveLockObjectData data2 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ "select * from mytable", conf);
Assert.assertEquals(data1, data2);
Assert.assertEquals(data1.hashCode(), data2.hashCode());
@@ -38,4 +43,25 @@ public class TestHiveLockObject {
Assert.assertEquals(obj1.hashCode(), obj2.hashCode());
}
+ @Test
+ public void testTruncate() {
+ conf.setIntVar(HiveConf.ConfVars.HIVE_LOCK_QUERY_STRING_MAX_LENGTH, 1000000);
+ HiveLockObjectData data0 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ "01234567890", conf);
+ Assert.assertEquals("With default settings query string should not be truncated",
+ data0.getQueryStr().length(), 11);
+ conf.setIntVar(HiveConf.ConfVars.HIVE_LOCK_QUERY_STRING_MAX_LENGTH, 10);
+ HiveLockObjectData data1 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ "01234567890", conf);
+ HiveLockObjectData data2 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ "0123456789", conf);
+ HiveLockObjectData data3 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ "012345678", conf);
+ HiveLockObjectData data4 = new HiveLockObjectData("ID1", "SHARED", "1997-07-01",
+ null, conf);
+ Assert.assertEquals("Long string truncation failed", data1.getQueryStr().length(), 10);
+ Assert.assertEquals("String truncation failed", data2.getQueryStr().length(), 10);
+ Assert.assertEquals("Short string should not be truncated", data3.getQueryStr().length(), 9);
+ Assert.assertNull("Null query string handling failed", data4.getQueryStr());
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0e4976/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
index 3f9926e..a7a76a4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
@@ -62,7 +62,7 @@ public class TestZookeeperLockManager {
@Before
public void setup() {
conf = new HiveConf();
- lockObjData = new HiveLockObjectData("1", "10", "SHARED", "show tables");
+ lockObjData = new HiveLockObjectData("1", "10", "SHARED", "show tables", conf);
hiveLock = new HiveLockObject(TABLE, lockObjData);
zLock = new ZooKeeperHiveLock(TABLE_LOCK_PATH, hiveLock, HiveLockMode.SHARED);