You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2017/07/12 21:15:14 UTC
[2/3] nifi git commit: NIFI-1613: ConvertJSONToSQL truncates numeric
value wrongly.
NIFI-1613: ConvertJSONToSQL truncates numeric value wrongly.
- Changed boolean value conversion to use Boolean.valueOf.
- Updated comments in source code to reflect current situation more clearly.
- Updated tests those have been added since the original commits were made.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/3844a821
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/3844a821
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/3844a821
Branch: refs/heads/master
Commit: 3844a821f18e6a76e44e289c923be333e49d5251
Parents: 3b2e43b
Author: Koji Kawamura <ij...@apache.org>
Authored: Wed Jul 5 14:49:32 2017 +0900
Committer: Matt Burgess <ma...@apache.org>
Committed: Wed Jul 12 17:02:55 2017 -0400
----------------------------------------------------------------------
.../processors/standard/ConvertJSONToSQL.java | 21 ++---
.../standard/TestConvertJSONToSQL.java | 90 --------------------
2 files changed, 6 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/3844a821/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
index ca92ad4..2f2123a 100755
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
@@ -514,19 +514,10 @@ public class ConvertJSONToSQL extends AbstractProcessor {
// only "true" is considered true, everything else is false
case Types.BOOLEAN:
- switch (fieldValue==null?"":fieldValue) {
- case "true":
- fieldValue = "true";
- break;
-
- default:
- fieldValue = "false";
- break;
- }
+ fieldValue = Boolean.valueOf(fieldValue).toString();
break;
// Don't truncate numeric types.
- // Should we check value is indeed number and throw error if not?
case Types.TINYINT:
case Types.SMALLINT:
case Types.INTEGER:
@@ -538,11 +529,11 @@ public class ConvertJSONToSQL extends AbstractProcessor {
case Types.NUMERIC:
break;
- // Don't truncate date and time types.
- // Should we check value is indeed correct date and/or time and throw error if not?
- // We assume date and time is already correct, but because ConvertJSONToSQL is often used together with PutSQL
- // maybe we should assure PutSQL correctly understands date and time values.
- // Currently PutSQL expect Long numeric values. But JSON usually uses ISO 8601, for example: 2012-04-23T18:25:43.511Z for dates.
+ // Don't truncate DATE, TIME and TIMESTAMP types. We assume date and time is already correct in long representation.
+ // Specifically, milliseconds since January 1, 1970, 00:00:00 GMT
+ // However, for TIMESTAMP, PutSQL accepts optional timestamp format via FlowFile attribute.
+ // See PutSQL.setParameter method and NIFI-3430 for detail.
+ // Alternatively, user can use JSONTreeReader and PutDatabaseRecord to handle date format more efficiently.
case Types.DATE:
case Types.TIME:
case Types.TIMESTAMP:
http://git-wip-us.apache.org/repos/asf/nifi/blob/3844a821/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java
index 3f9c52c..2cd66eb 100755
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java
@@ -106,18 +106,9 @@ public class TestConvertJSONToSQL {
@Test
public void testInsertQuotedIdentifiers() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "INSERT");
@@ -143,18 +134,9 @@ public class TestConvertJSONToSQL {
@Test
public void testInsertQuotedTableIdentifier() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "INSERT");
@@ -234,18 +216,9 @@ public class TestConvertJSONToSQL {
@Test
public void testUpdateQuotedTableIdentifier() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE");
@@ -297,18 +270,9 @@ public class TestConvertJSONToSQL {
@Test
public void testMultipleInsertsQuotedIdentifiers() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "INSERT");
@@ -359,18 +323,9 @@ public class TestConvertJSONToSQL {
@Test
public void testUpdateBasedOnPrimaryKeyQuotedIdentifier() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE");
@@ -449,18 +404,9 @@ public class TestConvertJSONToSQL {
@Test
public void testUpdateBasedOnUpdateKeyQuotedIdentifier() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE");
@@ -796,18 +742,9 @@ public class TestConvertJSONToSQL {
@Test
public void testDelete() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE");
@@ -831,18 +768,9 @@ public class TestConvertJSONToSQL {
@Test
public void testDeleteQuotedIdentifiers() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE");
@@ -868,18 +796,9 @@ public class TestConvertJSONToSQL {
@Test
public void testDeleteQuotedTableIdentifier() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE");
@@ -905,18 +824,9 @@ public class TestConvertJSONToSQL {
@Test
public void testDeleteWithNullValue() throws InitializationException, ProcessException, SQLException, IOException {
final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);
- final File tempDir = folder.getRoot();
- final File dbDir = new File(tempDir, "db");
- final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
runner.addControllerService("dbcp", service);
runner.enableControllerService(service);
- try (final Connection conn = service.getConnection()) {
- try (final Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(createPersons);
- }
- }
-
runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE");