[incubator-gobblin] branch master updated: [GOBBLIN-1146] Allow configuring autocommit in JDBCWriters

[hu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 5ece4e1  [GOBBLIN-1146] Allow configuring autocommit in JDBCWriters
5ece4e1 is described below

commit 5ece4e15140fb34a2a17f4b7b425c4252ca0ea02
Author: zhchen <zh...@linkedin.com>
AuthorDate: Tue May 12 16:52:51 2020 -0700

    [GOBBLIN-1146] Allow configuring autocommit in JDBCWriters
    
    Closes #2984 from zxcware/jdbc
---
 .../src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java | 2 +-
 .../src/main/java/org/apache/gobblin/writer/JdbcWriter.java            | 3 +++
 .../java/org/apache/gobblin/writer/commands/MySqlWriterCommands.java   | 2 --
 .../org/apache/gobblin/writer/commands/PostgresWriterCommands.java     | 2 --
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java b/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java
index d7d2de2..3ebba0e 100644
--- a/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java
+++ b/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java
@@ -274,7 +274,7 @@ public class PartitionedDataWriter<S, D> extends WriterWrapper<D> implements Fin
         entry.getValue().cleanup();
         writersCleanedUp++;
       } catch (Throwable throwable) {
-        log.error(String.format("Failed to cleanup writer for partition %s.", entry.getKey()));
+        log.error(String.format("Failed to cleanup writer for partition %s.", entry.getKey()), throwable);
       }
     }
     if (writersCleanedUp < this.partitionWriters.asMap().size()) {
diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/JdbcWriter.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/JdbcWriter.java
index b98f52f..7404f30 100644
--- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/JdbcWriter.java
+++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/JdbcWriter.java
@@ -67,6 +67,8 @@ import com.google.common.base.Preconditions;
 public class JdbcWriter implements DataWriter<JdbcEntryData> {
   private static final Logger LOG = LoggerFactory.getLogger(JdbcWriter.class);
 
+  public static final String ENABLE_AUTO_COMMIT = "jdbcWriter.enableAutoCommit";
+
   private final Connection conn;
   private final State state;
   private final JdbcWriterCommands commands;
@@ -91,6 +93,7 @@ public class JdbcWriter implements DataWriter<JdbcEntryData> {
         "Staging table is missing with key " + stagingTableKey);
     try {
       this.conn = createConnection();
+      this.conn.setAutoCommit(this.state.getPropAsBoolean(ENABLE_AUTO_COMMIT, false));
       this.commands = new JdbcWriterCommandsFactory().newInstance(this.state, this.conn);
       this.commands.setConnectionParameters(this.state.getProperties(), this.conn);
     } catch (SQLException e) {
diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/MySqlWriterCommands.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/MySqlWriterCommands.java
index 534e019..5c4da5a 100644
--- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/MySqlWriterCommands.java
+++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/MySqlWriterCommands.java
@@ -59,8 +59,6 @@ public class MySqlWriterCommands implements JdbcWriterCommands {
 
   @Override
   public void setConnectionParameters(Properties properties, Connection conn) throws SQLException {
-    // MySQL writer always uses one single transaction
-    this.conn.setAutoCommit(false);
   }
 
   @Override
diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/PostgresWriterCommands.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/PostgresWriterCommands.java
index 87e89c3..baa2b44 100644
--- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/PostgresWriterCommands.java
+++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/writer/commands/PostgresWriterCommands.java
@@ -59,8 +59,6 @@ public class PostgresWriterCommands implements JdbcWriterCommands {
   @Override
   public void setConnectionParameters(Properties properties, Connection conn)
       throws SQLException {
-    // Postgres writer always uses one single transaction
-    this.conn.setAutoCommit(false);
   }
 
   @Override