You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2020/05/09 17:24:39 UTC
[hive] branch master updated: HIVE-23350 : Upgrade DBCP To DBCP
2.7.0 (David Mollitor via Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 809a859 HIVE-23350 : Upgrade DBCP To DBCP 2.7.0 (David Mollitor via Ashutosh Chauhan)
809a859 is described below
commit 809a859f98b520f2989db6aff7bd5451bb546596
Author: David Mollitor <dm...@apache.org>
AuthorDate: Sat May 9 10:24:02 2020 -0700
HIVE-23350 : Upgrade DBCP To DBCP 2.7.0 (David Mollitor via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
itests/hive-unit/pom.xml | 6 --
jdbc-handler/pom.xml | 1 -
metastore/pom.xml | 10 ----
pom.xml | 10 ++--
standalone-metastore/metastore-common/pom.xml | 4 --
standalone-metastore/metastore-server/pom.xml | 4 +-
.../datasource/DbCPDataSourceProvider.java | 67 +++++++++++-----------
.../datasource/TestDataSourceProviderFactory.java | 3 +-
.../schematool/TestSchemaToolForMetastore.java | 2 +-
standalone-metastore/pom.xml | 12 ++--
10 files changed, 52 insertions(+), 67 deletions(-)
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index 2219002..026c465 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -431,12 +431,6 @@
<version>${plexus.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-dbcp</groupId>
- <artifactId>commons-dbcp</artifactId>
- <version>${commons-dbcp.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<profiles>
diff --git a/jdbc-handler/pom.xml b/jdbc-handler/pom.xml
index b2d9ec8..da137cd 100644
--- a/jdbc-handler/pom.xml
+++ b/jdbc-handler/pom.xml
@@ -89,7 +89,6 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
- <version>${commons-dbcp2.version}</version>
</dependency>
<dependency>
diff --git a/metastore/pom.xml b/metastore/pom.xml
index cca56a5..f1d5b66 100644
--- a/metastore/pom.xml
+++ b/metastore/pom.xml
@@ -87,16 +87,6 @@
<version>${derby.version}</version>
</dependency>
<dependency>
- <groupId>commons-pool</groupId>
- <artifactId>commons-pool</artifactId>
- <version>${commons-pool.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-dbcp</groupId>
- <artifactId>commons-dbcp</artifactId>
- <version>${commons-dbcp.version}</version>
- </dependency>
- <dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr-runtime</artifactId>
<version>${antlr.version}</version>
diff --git a/pom.xml b/pom.xml
index 2e1e9bb..1432bcf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -141,10 +141,7 @@
<commons-exec.version>1.1</commons-exec.version>
<commons-io.version>2.6</commons-io.version>
<commons-lang3.version>3.9</commons-lang3.version>
- <commons-pool.version>1.5.4</commons-pool.version>
- <commons-dbcp.version>1.4</commons-dbcp.version>
- <commons-pool2.version>2.6.1</commons-pool2.version>
- <commons-dbcp2.version>2.6.0</commons-dbcp2.version>
+ <commons-dbcp2.version>2.7.0</commons-dbcp2.version>
<commons-text.version>1.8</commons-text.version>
<derby.version>10.14.1.0</derby.version>
<dropwizard.version>3.1.0</dropwizard.version>
@@ -351,6 +348,11 @@
<version>${commons-io.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
+ <version>${commons-dbcp2.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty.version}</version>
diff --git a/standalone-metastore/metastore-common/pom.xml b/standalone-metastore/metastore-common/pom.xml
index ab26915..e8748c7 100644
--- a/standalone-metastore/metastore-common/pom.xml
+++ b/standalone-metastore/metastore-common/pom.xml
@@ -69,10 +69,6 @@
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
- <groupId>commons-dbcp</groupId>
- <artifactId>commons-dbcp</artifactId>
- </dependency>
- <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</dependency>
diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml
index 7355e93..021c25a 100644
--- a/standalone-metastore/metastore-server/pom.xml
+++ b/standalone-metastore/metastore-server/pom.xml
@@ -71,8 +71,8 @@
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
- <groupId>commons-dbcp</groupId>
- <artifactId>commons-dbcp</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java
index c687a25..1a5a1d2 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java
@@ -17,24 +17,26 @@
*/
package org.apache.hadoop.hive.metastore.datasource;
-import org.apache.commons.dbcp.BasicDataSource;
-import org.apache.commons.dbcp.ConnectionFactory;
-import org.apache.commons.dbcp.DataSourceConnectionFactory;
-import org.apache.commons.dbcp.PoolableConnectionFactory;
-import org.apache.commons.dbcp.PoolingDataSource;
-import org.apache.commons.pool.impl.GenericObjectPool;
+import static org.apache.hadoop.hive.metastore.DatabaseProduct.MYSQL;
+import static org.apache.hadoop.hive.metastore.DatabaseProduct.determineDatabaseProduct;
+
+import java.sql.SQLException;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.apache.commons.dbcp2.ConnectionFactory;
+import org.apache.commons.dbcp2.DataSourceConnectionFactory;
+import org.apache.commons.dbcp2.PoolableConnectionFactory;
+import org.apache.commons.dbcp2.PoolingDataSource;
+import org.apache.commons.pool2.impl.BaseObjectPoolConfig;
+import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.DatabaseProduct;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.sql.DataSource;
-import java.sql.SQLException;
-
-import static org.apache.hadoop.hive.metastore.DatabaseProduct.MYSQL;
-import static org.apache.hadoop.hive.metastore.DatabaseProduct.determineDatabaseProduct;
-
/**
* DataSourceProvider for the dbcp connection pool.
*/
@@ -55,6 +57,7 @@ public class DbCPDataSourceProvider implements DataSourceProvider {
private static final String CONNECTION_SOFT_MIN_EVICTABLE_IDLE_TIME = DBCP + ".softMinEvictableIdleTimeMillis";
private static final String CONNECTION_LIFO = DBCP + ".lifo";
+ @SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public DataSource create(Configuration hdpConfig) throws SQLException {
LOG.debug("Creating dbcp connection pool for the MetaStore");
@@ -67,6 +70,8 @@ public class DbCPDataSourceProvider implements DataSourceProvider {
dbcpDs.setUrl(driverUrl);
dbcpDs.setUsername(user);
dbcpDs.setPassword(passwd);
+ dbcpDs.setDefaultReadOnly(false);
+ dbcpDs.setDefaultAutoCommit(true);
DatabaseProduct dbProduct = determineDatabaseProduct(driverUrl);
switch (dbProduct){
@@ -77,31 +82,36 @@ public class DbCPDataSourceProvider implements DataSourceProvider {
case POSTGRES:
dbcpDs.setConnectionProperties("reWriteBatchedInserts=true");
break;
+ default:
+ break;
}
int maxPoolSize = hdpConfig.getInt(
MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.getVarname(),
((Long) MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.getDefaultVal()).intValue());
long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
- int connectionMaxIlde = hdpConfig.getInt(CONNECTION_MAX_IDLE_PROPERTY, GenericObjectPool.DEFAULT_MAX_IDLE);
- int connectionMinIlde = hdpConfig.getInt(CONNECTION_MIN_IDLE_PROPERTY, GenericObjectPool.DEFAULT_MIN_IDLE);
+ int connectionMaxIlde = hdpConfig.getInt(CONNECTION_MAX_IDLE_PROPERTY, 8);
+ int connectionMinIlde = hdpConfig.getInt(CONNECTION_MIN_IDLE_PROPERTY, 0);
boolean testOnBorrow = hdpConfig.getBoolean(CONNECTION_TEST_BORROW_PROPERTY,
- GenericObjectPool.DEFAULT_TEST_ON_BORROW);
+ BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW);
long evictionTimeMillis = hdpConfig.getLong(CONNECTION_MIN_EVICT_MILLIS_PROPERTY,
- GenericObjectPool.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS);
+ BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS);
boolean testWhileIdle = hdpConfig.getBoolean(CONNECTION_TEST_IDLEPROPERTY,
- GenericObjectPool.DEFAULT_TEST_WHILE_IDLE);
+ BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE);
long timeBetweenEvictionRuns = hdpConfig.getLong(CONNECTION_TIME_BETWEEN_EVICTION_RUNS_MILLIS,
- GenericObjectPool.DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS);
+ BaseObjectPoolConfig.DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS);
int numTestsPerEvictionRun = hdpConfig.getInt(CONNECTION_NUM_TESTS_PER_EVICTION_RUN,
- GenericObjectPool.DEFAULT_NUM_TESTS_PER_EVICTION_RUN);
- boolean testOnReturn = hdpConfig.getBoolean(CONNECTION_TEST_ON_RETURN, GenericObjectPool.DEFAULT_TEST_ON_RETURN);
+ BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN);
+ boolean testOnReturn = hdpConfig.getBoolean(CONNECTION_TEST_ON_RETURN, BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN);
long softMinEvictableIdleTimeMillis = hdpConfig.getLong(CONNECTION_SOFT_MIN_EVICTABLE_IDLE_TIME,
- GenericObjectPool.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS);
- boolean lifo = hdpConfig.getBoolean(CONNECTION_LIFO, GenericObjectPool.DEFAULT_LIFO);
+ BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS);
+ boolean lifo = hdpConfig.getBoolean(CONNECTION_LIFO, BaseObjectPoolConfig.DEFAULT_LIFO);
- GenericObjectPool objectPool = new GenericObjectPool();
- objectPool.setMaxActive(maxPoolSize);
- objectPool.setMaxWait(connectionTimeout);
+ ConnectionFactory connFactory = new DataSourceConnectionFactory(dbcpDs);
+ PoolableConnectionFactory poolableConnFactory = new PoolableConnectionFactory(connFactory, null);
+
+ GenericObjectPool objectPool = new GenericObjectPool(poolableConnFactory);
+ objectPool.setMaxTotal(maxPoolSize);
+ objectPool.setMaxWaitMillis(connectionTimeout);
objectPool.setMaxIdle(connectionMaxIlde);
objectPool.setMinIdle(connectionMinIlde);
objectPool.setTestOnBorrow(testOnBorrow);
@@ -113,13 +123,6 @@ public class DbCPDataSourceProvider implements DataSourceProvider {
objectPool.setSoftMinEvictableIdleTimeMillis(softMinEvictableIdleTimeMillis);
objectPool.setLifo(lifo);
- ConnectionFactory connFactory = new DataSourceConnectionFactory(dbcpDs);
- // This doesn't get used, but it's still necessary, see
- // https://git1-us-west.apache.org/repos/asf?p=commons-dbcp.git;a=blob;f=doc/ManualPoolingDataSourceExample.java;
- // h=f45af2b8481f030b27364e505984c0eef4f35cdb;hb=refs/heads/DBCP_1_5_x_BRANCH
- PoolableConnectionFactory poolableConnFactory =
- new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true);
-
if (dbProduct == MYSQL) {
poolableConnFactory.setValidationQuery("SET @@session.sql_mode=ANSI_QUOTES");
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
index 43ca0ef..33955ed 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.metastore.datasource;
import com.zaxxer.hikari.HikariDataSource;
-import org.apache.commons.dbcp.PoolingDataSource;
+
+import org.apache.commons.dbcp2.PoolingDataSource;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
index 11099a6..709c90c 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
@@ -31,7 +31,7 @@ import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.util.Random;
-import org.apache.commons.dbcp.DelegatingConnection;
+import org.apache.commons.dbcp2.DelegatingConnection;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.text.StrTokenizer;
import org.apache.hadoop.conf.Configuration;
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 9b57c59..f91e5f3 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -69,9 +69,9 @@
<!-- Dependency versions -->
<antlr.version>3.5.2</antlr.version>
<apache-directory-server.version>1.5.7</apache-directory-server.version>
- <commons-dbcp.version>1.4</commons-dbcp.version>
<commons-lang3.version>3.9</commons-lang3.version>
<commons-logging.version>1.1.3</commons-logging.version>
+ <commons-dbcp2.version>2.7.0</commons-dbcp2.version>
<datanucleus-api-jdo.version>4.2.4</datanucleus-api-jdo.version>
<datanucleus-core.version>4.1.17</datanucleus-core.version>
<datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version>
@@ -147,11 +147,6 @@
<version>${hikaricp.version}</version>
</dependency>
<dependency>
- <groupId>commons-dbcp</groupId>
- <artifactId>commons-dbcp</artifactId>
- <version>${commons-dbcp.version}</version>
- </dependency>
- <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${dropwizard.version}</version>
@@ -245,6 +240,11 @@
<version>${storage-api.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
+ <version>${commons-dbcp2.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j2.version}</version>