[gobblin] branch master updated: Specifically name each Hikari connection pool created, for traceability (#3627)

[wl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wlo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new a1be38bf6 Specifically name each Hikari connection pool created, for traceability (#3627)
a1be38bf6 is described below

commit a1be38bf65526f1cc8ca151536831e981b183da1
Author: Kip Kohn <ck...@linkedin.com>
AuthorDate: Mon Jan 23 10:08:51 2023 -0800

    Specifically name each Hikari connection pool created, for traceability (#3627)
---
 .../gobblin/metastore/JobHistoryDataSourceProvider.java  |  3 +++
 .../org/apache/gobblin/metastore/MysqlStateStore.java    | 16 +++++++++++++++-
 .../org/apache/gobblin/source/jdbc/JdbcProvider.java     |  4 ++++
 .../service/modules/db/ServiceDatabaseProviderImpl.java  |  3 +++
 .../org/apache/gobblin/util/jdbc/DataSourceProvider.java |  3 +++
 5 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/JobHistoryDataSourceProvider.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/JobHistoryDataSourceProvider.java
index edf35d637..c175b352e 100644
--- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/JobHistoryDataSourceProvider.java
+++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/JobHistoryDataSourceProvider.java
@@ -19,6 +19,7 @@ package org.apache.gobblin.metastore;
 
 import java.time.Duration;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,9 +37,11 @@ import org.apache.gobblin.util.jdbc.MysqlDataSourceUtils;
  */
 public class JobHistoryDataSourceProvider extends org.apache.gobblin.util.jdbc.DataSourceProvider {
   private static final Logger LOG = LoggerFactory.getLogger(JobHistoryDataSourceProvider.class);
+  private static final AtomicInteger POOL_NUM = new AtomicInteger(0);
 
   @Inject
   public JobHistoryDataSourceProvider(@Named("dataSourceProperties") Properties properties) {
+    this.dataSource.setPoolName("HikariPool-" + POOL_NUM.incrementAndGet() + "-" + getClass().getSimpleName());
     this.dataSource.setDriverClassName(properties.getProperty(ConfigurationKeys.JOB_HISTORY_STORE_JDBC_DRIVER_KEY,
         ConfigurationKeys.DEFAULT_JOB_HISTORY_STORE_JDBC_DRIVER));
 
diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
index 3bbf8747b..c2bc1e68a 100644
--- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
+++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
@@ -26,6 +26,8 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.sql.Blob;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -36,6 +38,7 @@ import java.time.Duration;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 import javax.sql.DataSource;
@@ -79,6 +82,7 @@ import org.apache.gobblin.util.jdbc.MysqlDataSourceUtils;
  **/
 public class MysqlStateStore<T extends State> implements StateStore<T> {
   private static final Logger LOG = LoggerFactory.getLogger(MysqlStateStore.class);
+  private static final AtomicInteger POOL_NUM = new AtomicInteger(0);
 
   /** Specifies which 'Job State' query columns receive search evaluation (with SQL `LIKE` operator). */
   protected enum JobStateSearchColumns {
@@ -201,6 +205,16 @@ public class MysqlStateStore<T extends State> implements StateStore<T> {
     HikariDataSource dataSource = new HikariDataSource();
     PasswordManager passwordManager = PasswordManager.getInstance(ConfigUtils.configToProperties(config));
 
+    String jdbcUrl = config.getString(ConfigurationKeys.STATE_STORE_DB_URL_KEY);
+    String poolName = "HikariPool-" + POOL_NUM.incrementAndGet() + "-" + MysqlStateStore.class.getSimpleName();
+    try {
+      String dbPath = new URI(new URI(jdbcUrl).getSchemeSpecificPart()).getPath().replaceAll("\\W", "-");
+      // when possible, attempt discernment to the DB level
+      poolName += dbPath; // as the path will begin w/ "/", following `replaceAll`, no need to prepend additional "-"
+    } catch (URISyntaxException e) {
+      LOG.warn("unable to parse JDBC URL '{}' - {}", jdbcUrl, e.getMessage());
+    }
+    dataSource.setPoolName(poolName);
     dataSource.setDriverClassName(ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_JDBC_DRIVER_KEY,
         ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER));
     // MySQL server can timeout a connection so need to validate connections before use
@@ -213,7 +227,7 @@ public class MysqlStateStore<T extends State> implements StateStore<T> {
     dataSource.setConnectionTestQuery(validationQuery);
     dataSource.setAutoCommit(false);
     dataSource.setIdleTimeout(Duration.ofSeconds(60).toMillis());
-    dataSource.setJdbcUrl(config.getString(ConfigurationKeys.STATE_STORE_DB_URL_KEY));
+    dataSource.setJdbcUrl(jdbcUrl);
     // TODO: revisit following verification of successful connection pool migration:
     //   whereas `o.a.commons.dbcp.BasicDataSource` defaults min idle conns to 0, hikari defaults to 10.
     //   perhaps non-zero would have desirable runtime perf, but anything >0 currently fails unit tests (even 1!);
diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/JdbcProvider.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/JdbcProvider.java
index db6494088..e92080b29 100644
--- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/JdbcProvider.java
+++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/JdbcProvider.java
@@ -18,6 +18,7 @@
 package org.apache.gobblin.source.jdbc;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.gobblin.tunnel.Tunnel;
 
@@ -30,6 +31,8 @@ import com.zaxxer.hikari.HikariDataSource;
  * @author nveeramr
  */
 public class JdbcProvider extends HikariDataSource {
+  private static final AtomicInteger POOL_NUM = new AtomicInteger(0);
+
   private Tunnel tunnel;
 
   // If extract type is not provided then consider it as a default type
@@ -73,6 +76,7 @@ public class JdbcProvider extends HikariDataSource {
       }
     }
 
+    this.setPoolName("HikariPool-" + POOL_NUM.incrementAndGet() + "-" + getClass().getSimpleName());
     this.setDriverClassName(driver);
     this.setUsername(user);
     this.setPassword(password);
diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/db/ServiceDatabaseProviderImpl.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/db/ServiceDatabaseProviderImpl.java
index 7d2d719da..17431ea5f 100644
--- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/db/ServiceDatabaseProviderImpl.java
+++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/db/ServiceDatabaseProviderImpl.java
@@ -19,6 +19,7 @@ package org.apache.gobblin.service.modules.db;
 
 import java.time.Duration;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,6 +42,7 @@ import org.apache.gobblin.util.ConfigUtils;
 
 public class ServiceDatabaseProviderImpl implements ServiceDatabaseProvider {
   private static final Logger LOG = LoggerFactory.getLogger(ServiceDatabaseProviderImpl.class);
+  private static final AtomicInteger POOL_NUM = new AtomicInteger(0);
 
   private final Configuration configuration;
   private HikariDataSource dataSource;
@@ -62,6 +64,7 @@ public class ServiceDatabaseProviderImpl implements ServiceDatabaseProvider {
 
     dataSource = new HikariDataSource();
 
+    dataSource.setPoolName("HikariPool-" + POOL_NUM.incrementAndGet() + "-" + getClass().getSimpleName());
     dataSource.setJdbcUrl(configuration.getUrl());
     dataSource.setUsername(configuration.getUserName());
     dataSource.setPassword(configuration.getPassword());
diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/jdbc/DataSourceProvider.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/jdbc/DataSourceProvider.java
index d7f6596f0..53ee08f2f 100644
--- a/gobblin-utility/src/main/java/org/apache/gobblin/util/jdbc/DataSourceProvider.java
+++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/jdbc/DataSourceProvider.java
@@ -20,6 +20,7 @@ package org.apache.gobblin.util.jdbc;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.sql.DataSource;
 
@@ -39,6 +40,7 @@ import org.apache.gobblin.password.PasswordManager;
  */
 public class DataSourceProvider implements Provider<DataSource> {
   private static final Logger LOG = LoggerFactory.getLogger(DataSourceProvider.class);
+  private static final AtomicInteger POOL_NUM = new AtomicInteger(0);
 
   public static final String GOBBLIN_UTIL_JDBC_PREFIX = "gobblin.util.jdbc.";
   public static final String CONN_DRIVER = GOBBLIN_UTIL_JDBC_PREFIX + "conn.driver";
@@ -55,6 +57,7 @@ public class DataSourceProvider implements Provider<DataSource> {
   @Inject
   public DataSourceProvider(@Named("dataSourceProperties") Properties properties) {
     this.dataSource = new HikariDataSource();
+    this.dataSource.setPoolName("HikariPool-" + POOL_NUM.incrementAndGet() + "-" + getClass().getSimpleName());
     this.dataSource.setDriverClassName(properties.getProperty(CONN_DRIVER, DEFAULT_CONN_DRIVER));
     // the validation query should work beyond mysql; still, to bypass for any reason, heed directive
     if (!Boolean.parseBoolean(properties.getProperty(SKIP_VALIDATION_QUERY, "false"))) {