You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2019/04/17 12:30:32 UTC
[flink] branch master updated (e42f6b5 -> 8a91b07)
This is an automated email from the ASF dual-hosted git repository.
fhueske pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from e42f6b5 [FLINK-12170] [table-planner-blink] Add support for generating optimized logical plan for Over aggregate (#8157)
new 494f5be [FLINK-12198][jdbc] Add support for setting auto-commit mode of JDBCInputFormat DB connection.
new 3b1c14f [FLINK-12204][jdbc] Improve JDBCOutputFormat ClassCastException.
new 3422308 [hotfix][yarn] Fix typo in comment of YarnConfigurationITCase.
new 8a91b07 [hotfix][rockdb] Fix a swallowed exception cause in RocksDBListState.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../flink/api/java/io/jdbc/JDBCInputFormat.java | 19 ++++++++++++
.../flink/api/java/io/jdbc/JDBCOutputFormat.java | 8 +++--
.../flink/api/java/io/jdbc/JDBCFullTest.java | 19 ++++++++++++
.../api/java/io/jdbc/JDBCInputFormatTest.java | 35 ++++++++++++++++++++++
.../flink/api/java/io/jdbc/JDBCTestBase.java | 5 ++++
.../contrib/streaming/state/RocksDBListState.java | 2 +-
.../apache/flink/yarn/YarnConfigurationITCase.java | 2 +-
7 files changed, 86 insertions(+), 4 deletions(-)
[flink] 01/04: [FLINK-12198][jdbc] Add support for setting
auto-commit mode of JDBCInputFormat DB connection.
Posted by fh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fhueske pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 494f5be09d165c5adca092ef1bdcfec1a238c316
Author: Konstantinos Papadopoulos <Ko...@IRIworldwide.com>
AuthorDate: Tue Apr 16 13:55:56 2019 +0300
[FLINK-12198][jdbc] Add support for setting auto-commit mode of JDBCInputFormat DB connection.
This closes #8186.
---
.../flink/api/java/io/jdbc/JDBCInputFormat.java | 19 ++++++++++++
.../api/java/io/jdbc/JDBCInputFormatTest.java | 35 ++++++++++++++++++++++
2 files changed, 54 insertions(+)
diff --git a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
index 7d08814..cba125e 100644
--- a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
+++ b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
@@ -116,6 +116,8 @@ public class JDBCInputFormat extends RichInputFormat<Row, InputSplit> implements
private transient PreparedStatement statement;
private transient ResultSet resultSet;
private int fetchSize;
+ // Boolean to distinguish between default value and explicitly set autoCommit mode.
+ private Boolean autoCommit;
private boolean hasNext;
private Object[][] parameterValues;
@@ -143,6 +145,13 @@ public class JDBCInputFormat extends RichInputFormat<Row, InputSplit> implements
} else {
dbConn = DriverManager.getConnection(dbURL, username, password);
}
+
+ // set autoCommit mode only if it was explicitly configured.
+ // keep connection default otherwise.
+ if (autoCommit != null) {
+ dbConn.setAutoCommit(autoCommit);
+ }
+
statement = dbConn.prepareStatement(queryTemplate, resultSetType, resultSetConcurrency);
if (fetchSize == Integer.MIN_VALUE || fetchSize > 0) {
statement.setFetchSize(fetchSize);
@@ -323,6 +332,11 @@ public class JDBCInputFormat extends RichInputFormat<Row, InputSplit> implements
return statement;
}
+ @VisibleForTesting
+ Connection getDbConn() {
+ return dbConn;
+ }
+
/**
* A builder used to set parameters to the output format's configuration in a fluent way.
* @return builder
@@ -396,6 +410,11 @@ public class JDBCInputFormat extends RichInputFormat<Row, InputSplit> implements
return this;
}
+ public JDBCInputFormatBuilder setAutoCommit(Boolean autoCommit) {
+ format.autoCommit = autoCommit;
+ return this;
+ }
+
public JDBCInputFormat finish() {
if (format.username == null) {
LOG.info("Username was not supplied separately.");
diff --git a/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java b/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
index 10e8c66..81320e6 100644
--- a/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
+++ b/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
@@ -155,6 +155,41 @@ public class JDBCInputFormatTest extends JDBCTestBase {
}
@Test
+ public void testDefaultAutoCommitIsUsedIfNotConfiguredOtherwise() throws SQLException, ClassNotFoundException {
+
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername(DRIVER_CLASS)
+ .setDBUrl(DB_URL)
+ .setQuery(SELECT_ALL_BOOKS)
+ .setRowTypeInfo(ROW_TYPE_INFO)
+ .finish();
+ jdbcInputFormat.openInputFormat();
+
+ Class.forName(DRIVER_CLASS);
+ final boolean defaultAutoCommit = DriverManager.getConnection(DB_URL).getAutoCommit();
+
+ Assert.assertEquals(defaultAutoCommit, jdbcInputFormat.getDbConn().getAutoCommit());
+
+ }
+
+ @Test
+ public void testAutoCommitCanBeConfigured() throws SQLException {
+
+ final boolean desiredAutoCommit = false;
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername(DRIVER_CLASS)
+ .setDBUrl(DB_URL)
+ .setQuery(SELECT_ALL_BOOKS)
+ .setRowTypeInfo(ROW_TYPE_INFO)
+ .setAutoCommit(desiredAutoCommit)
+ .finish();
+
+ jdbcInputFormat.openInputFormat();
+ Assert.assertEquals(desiredAutoCommit, jdbcInputFormat.getDbConn().getAutoCommit());
+
+ }
+
+ @Test
public void testJDBCInputFormatWithoutParallelism() throws IOException {
jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
.setDrivername(DRIVER_CLASS)
[flink] 02/04: [FLINK-12204][jdbc] Improve JDBCOutputFormat
ClassCastException.
Posted by fh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fhueske pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3b1c14f9cbc3f4db0c4c14a8ba55edbccef76729
Author: hequn8128 <ch...@gmail.com>
AuthorDate: Tue Apr 16 15:07:43 2019 +0800
[FLINK-12204][jdbc] Improve JDBCOutputFormat ClassCastException.
This closes #8182.
---
.../flink/api/java/io/jdbc/JDBCOutputFormat.java | 8 ++++++--
.../apache/flink/api/java/io/jdbc/JDBCFullTest.java | 19 +++++++++++++++++++
.../apache/flink/api/java/io/jdbc/JDBCTestBase.java | 5 +++++
3 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
index 444485c..f773635 100644
--- a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
+++ b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
@@ -198,8 +198,12 @@ public class JDBCOutputFormat extends RichOutputFormat<Row> {
// case java.sql.Types.STRUC
}
} catch (ClassCastException e) {
- throw new RuntimeException(
- "Field index: " + index + ", field value: " + row.getField(index) + " " + e.getMessage(), e);
+ // enrich the exception with detailed information.
+ String errorMessage = String.format(
+ "%s, field index: %s, field value: %s.", e.getMessage(), index, row.getField(index));
+ ClassCastException enrichedException = new ClassCastException(errorMessage);
+ enrichedException.setStackTrace(e.getStackTrace());
+ throw enrichedException;
}
}
}
diff --git a/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCFullTest.java b/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCFullTest.java
index c1f2b25..51d39b2 100644
--- a/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCFullTest.java
+++ b/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCFullTest.java
@@ -50,6 +50,25 @@ public class JDBCFullTest extends JDBCTestBase {
runTest(true);
}
+ @Test
+ public void testEnrichedClassCastException() throws Exception {
+ exception.expect(ClassCastException.class);
+ exception.expectMessage(
+ "java.lang.String cannot be cast to java.lang.Double, field index: 3, field value: 11.11.");
+
+ JDBCOutputFormat jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername(JDBCTestBase.DRIVER_CLASS)
+ .setDBUrl(JDBCTestBase.DB_URL)
+ .setQuery("insert into newbooks (id, title, author, price, qty) values (?,?,?,?,?)")
+ .setSqlTypes(new int[]{Types.INTEGER, Types.VARCHAR, Types.VARCHAR, Types.DOUBLE, Types.INTEGER})
+ .finish();
+
+ jdbcOutputFormat.open(1, 1);
+ Row inputRow = Row.of(1001, "Java public for dummies", "Tan Ah Teck", "11.11", 11);
+ jdbcOutputFormat.writeRecord(inputRow);
+ jdbcOutputFormat.close();
+ }
+
private void runTest(boolean exploitParallelism) throws Exception {
ExecutionEnvironment environment = ExecutionEnvironment.getExecutionEnvironment();
JDBCInputFormatBuilder inputBuilder = JDBCInputFormat.buildJDBCInputFormat()
diff --git a/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCTestBase.java b/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCTestBase.java
index 1d41d37..febbbd3 100644
--- a/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCTestBase.java
+++ b/flink-connectors/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCTestBase.java
@@ -23,6 +23,8 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
import java.io.OutputStream;
import java.sql.Connection;
@@ -35,6 +37,9 @@ import java.sql.Statement;
*/
public class JDBCTestBase {
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
public static final String DRIVER_CLASS = "org.apache.derby.jdbc.EmbeddedDriver";
public static final String DB_URL = "jdbc:derby:memory:ebookshop";
public static final String INPUT_TABLE = "books";
[flink] 04/04: [hotfix][rockdb] Fix a swallowed exception cause in
RocksDBListState.
Posted by fh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fhueske pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 8a91b07d5ab85b6bf100a36c5e26e6297a7f3a2a
Author: blueszheng <ki...@163.com>
AuthorDate: Thu Mar 14 16:21:56 2019 +0800
[hotfix][rockdb] Fix a swallowed exception cause in RocksDBListState.
This closes #7983.
---
.../java/org/apache/flink/contrib/streaming/state/RocksDBListState.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java
index 00123f5..c18adb1 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java
@@ -148,7 +148,7 @@ class RocksDBListState<K, N, V>
return element;
}
} catch (IOException e) {
- throw new FlinkRuntimeException("Unexpected list element deserialization failure");
+ throw new FlinkRuntimeException("Unexpected list element deserialization failure", e);
}
return null;
}
[flink] 03/04: [hotfix][yarn] Fix typo in comment of
YarnConfigurationITCase.
Posted by fh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fhueske pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3422308f542222d4d4ce64231ba75ff39764bc8e
Author: Paul Lam <pa...@gmail.com>
AuthorDate: Wed Mar 20 16:36:18 2019 +0800
[hotfix][yarn] Fix typo in comment of YarnConfigurationITCase.
This closes #8161.
---
.../src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
index a2772b7..6da6055 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
@@ -181,7 +181,7 @@ public class YarnConfigurationITCase extends YarnTestBase {
final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;
// We compare here physical memory assigned to a container with the heap memory that we should pass to
- // jvm as Xmx parameter. Those value might differ significantly due to sytem page size or jvm
+ // jvm as Xmx parameter. Those value might differ significantly due to system page size or jvm
// implementation therefore we use 15% threshold here.
assertThat(
(double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize,