You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2022/03/08 19:35:04 UTC
[hive] branch branch-3 updated: HIVE-25522: NullPointerException in TxnHandler (#2647) (#3062)
This is an automated email from the ASF dual-hosted git repository.
szehon pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 01924c8 HIVE-25522: NullPointerException in TxnHandler (#2647) (#3062)
01924c8 is described below
commit 01924c8d54b146dee44c30d104fb60fbcdec0e87
Author: Szehon Ho <sz...@gmail.com>
AuthorDate: Tue Mar 8 11:34:21 2022 -0800
HIVE-25522: NullPointerException in TxnHandler (#2647) (#3062)
---
.../hadoop/hive/metastore/txn/TxnHandler.java | 94 ++++++++++++----------
1 file changed, 52 insertions(+), 42 deletions(-)
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index b969efb..e6750a5 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -296,14 +296,15 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
checkQFileTestHack();
synchronized (TxnHandler.class) {
- if (connPool == null) {
- Connection dbConn = null;
- // Set up the JDBC connection pool
- try {
- int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
- long getConnectionTimeoutMs = 30000;
+ int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
+ long getConnectionTimeoutMs = 30000;
+ synchronized (TxnHandler.class) {
+ if (connPool == null) {
connPool = setupJdbcConnectionPool(conf, maxPoolSize, getConnectionTimeoutMs);
- /*the mutex pools should ideally be somewhat larger since some operations require 1
+ }
+
+ if (connPoolMutex == null) {
+ /*the mutex pools should ideally be somewhat larger since some operations require 1
connection from each pool and we want to avoid taking a connection from primary pool
and then blocking because mutex pool is empty. There is only 1 thread in any HMS trying
to mutex on each MUTEX_KEY except MUTEX_KEY.CheckLock. The CheckLock operation gets a
@@ -311,15 +312,19 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
order (not very elegant...). So number of connection requests for connPoolMutex cannot
exceed (size of connPool + MUTEX_KEY.values().length - 1).*/
connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize + MUTEX_KEY.values().length, getConnectionTimeoutMs);
- dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
- determineDatabaseProduct(dbConn);
+ }
+
+ if (dbProduct == null) {
+ try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED)) {
+ determineDatabaseProduct(dbConn);
+ } catch (SQLException e) {
+ LOG.error("Unable to determine database product", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ if (sqlGenerator == null) {
sqlGenerator = new SQLGenerator(dbProduct, conf);
- } catch (SQLException e) {
- String msg = "Unable to instantiate JDBC connection pooling, " + e.getMessage();
- LOG.error(msg);
- throw new RuntimeException(e);
- } finally {
- closeDbConn(dbConn);
}
}
}
@@ -4750,33 +4755,38 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
}
- private static synchronized DataSource setupJdbcConnectionPool(Configuration conf, int maxPoolSize, long getConnectionTimeoutMs) throws SQLException {
- String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(conf);
- String user = DataSourceProvider.getMetastoreJdbcUser(conf);
- String passwd = DataSourceProvider.getMetastoreJdbcPasswd(conf);
- String connectionPooler = MetastoreConf.getVar(conf, ConfVars.CONNECTION_POOLING_TYPE).toLowerCase();
-
- if ("bonecp".equals(connectionPooler)) {
- doRetryOnConnPool = true; // Enable retries to work around BONECP bug.
- return new BoneCPDataSourceProvider().create(conf);
- } else if ("dbcp".equals(connectionPooler)) {
- GenericObjectPool objectPool = new GenericObjectPool();
- //https://commons.apache.org/proper/commons-pool/api-1.6/org/apache/commons/pool/impl/GenericObjectPool.html#setMaxActive(int)
- objectPool.setMaxActive(maxPoolSize);
- objectPool.setMaxWait(getConnectionTimeoutMs);
- ConnectionFactory connFactory = new DriverManagerConnectionFactory(driverUrl, user, passwd);
- // This doesn't get used, but it's still necessary, see
- // http://svn.apache.org/viewvc/commons/proper/dbcp/branches/DBCP_1_4_x_BRANCH/doc/ManualPoolingDataSourceExample.java?view=markup
- PoolableConnectionFactory poolConnFactory =
- new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true);
- return new PoolingDataSource(objectPool);
- } else if ("hikaricp".equals(connectionPooler)) {
- return new HikariCPDataSourceProvider().create(conf);
- } else if ("none".equals(connectionPooler)) {
- LOG.info("Choosing not to pool JDBC connections");
- return new NoPoolConnectionPool(conf);
- } else {
- throw new RuntimeException("Unknown JDBC connection pooling " + connectionPooler);
+ private static synchronized DataSource setupJdbcConnectionPool(Configuration conf, int maxPoolSize, long getConnectionTimeoutMs) {
+ try {
+ String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(conf);
+ String user = DataSourceProvider.getMetastoreJdbcUser(conf);
+ String passwd = DataSourceProvider.getMetastoreJdbcPasswd(conf);
+ String connectionPooler = MetastoreConf.getVar(conf, ConfVars.CONNECTION_POOLING_TYPE).toLowerCase();
+
+ if ("bonecp".equals(connectionPooler)) {
+ doRetryOnConnPool = true; // Enable retries to work around BONECP bug.
+ return new BoneCPDataSourceProvider().create(conf);
+ } else if ("dbcp".equals(connectionPooler)) {
+ GenericObjectPool objectPool = new GenericObjectPool();
+ //https://commons.apache.org/proper/commons-pool/api-1.6/org/apache/commons/pool/impl/GenericObjectPool.html#setMaxActive(int)
+ objectPool.setMaxActive(maxPoolSize);
+ objectPool.setMaxWait(getConnectionTimeoutMs);
+ ConnectionFactory connFactory = new DriverManagerConnectionFactory(driverUrl, user, passwd);
+ // This doesn't get used, but it's still necessary, see
+ // http://svn.apache.org/viewvc/commons/proper/dbcp/branches/DBCP_1_4_x_BRANCH/doc/ManualPoolingDataSourceExample.java?view=markup
+ PoolableConnectionFactory poolConnFactory =
+ new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true);
+ return new PoolingDataSource(objectPool);
+ } else if ("hikaricp".equals(connectionPooler)) {
+ return new HikariCPDataSourceProvider().create(conf);
+ } else if ("none".equals(connectionPooler)) {
+ LOG.info("Choosing not to pool JDBC connections");
+ return new NoPoolConnectionPool(conf);
+ } else {
+ throw new RuntimeException("Unknown JDBC connection pooling " + connectionPooler);
+ }
+ } catch (SQLException e) {
+ LOG.error("Unable to instantiate JDBC connection pooling", e);
+ throw new RuntimeException(e);
}
}