You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by da...@apache.org on 2015/06/21 19:23:11 UTC

[10/29] incubator-nifi git commit: NIFI-626: Code cleanup to adhere to NiFi coding styles

NIFI-626: Code cleanup to adhere to NiFi coding styles


Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/a3b8e44a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/a3b8e44a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/a3b8e44a

Branch: refs/heads/NIFI-632
Commit: a3b8e44ad5aff7b68dde26c242ba171d1884b948
Parents: d98757e
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Jun 10 12:16:52 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Jun 17 12:37:25 2015 -0400

----------------------------------------------------------------------
 .../nifi/processors/standard/ExecuteSQL.java    |  71 +++--
 .../processors/standard/util/JdbcCommon.java    | 181 +++++++------
 .../processors/standard/TestExecuteSQL.java     |  57 ++--
 .../standard/util/TestJdbcHugeStream.java       | 259 +++++++++----------
 nifi/pom.xml                                    |   5 +
 5 files changed, 280 insertions(+), 293 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3b8e44a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
index 07e6fe9..6647c4c 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
@@ -44,6 +44,7 @@ import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.OutputStreamCallback;
 import org.apache.nifi.processor.util.StandardValidators;
 import org.apache.nifi.processors.standard.util.JdbcCommon;
+import org.apache.nifi.util.LongHolder;
 import org.apache.nifi.util.StopWatch;
 
 @EventDriven
@@ -62,7 +63,7 @@ public class ExecuteSQL extends AbstractProcessor {
     		.description("SQL query execution failed. Incoming FlowFile will be penalized and routed to this relationship")
     		.build();
     private final Set<Relationship> relationships;
-    
+
     public static final PropertyDescriptor DBCP_SERVICE = new PropertyDescriptor.Builder()
     		.name("Database Connection Pooling Service")
     		.description("The Controller Service that is used to obtain connection to database")
@@ -91,11 +92,11 @@ public class ExecuteSQL extends AbstractProcessor {
     private final List<PropertyDescriptor> propDescriptors;
 
     public ExecuteSQL() {
-        HashSet<Relationship> r = new HashSet<>();
+        final Set<Relationship> r = new HashSet<>();
         r.add(REL_SUCCESS);
         relationships = Collections.unmodifiableSet(r);
 
-        ArrayList<PropertyDescriptor> pds = new ArrayList<>();
+        final List<PropertyDescriptor> pds = new ArrayList<>();
         pds.add(DBCP_SERVICE);
         pds.add(SQL_SELECT_QUERY);
         pds.add(QUERY_TIMEOUT);
@@ -113,7 +114,7 @@ public class ExecuteSQL extends AbstractProcessor {
     }
 
     @Override
-    public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
         FlowFile incoming = session.get();
         if (incoming == null) {
             return;
@@ -126,44 +127,30 @@ public class ExecuteSQL extends AbstractProcessor {
         final Integer queryTimeout = context.getProperty(QUERY_TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue();
 
 		final StopWatch stopWatch = new StopWatch(true);
-        
-        try {
-			final Connection con = dbcpService.getConnection();
-			try {
-				final Statement st = con.createStatement();
-				try {
-					st.setQueryTimeout(queryTimeout);	// timeout in seconds
-					FlowFile outgoing = session.write(incoming, new OutputStreamCallback() {
-						@Override
-						public void process(final OutputStream out) throws IOException {
-							try {
-								logger.info("start executing query {}", new Object[]{selectQuery});
-								ResultSet resultSet = st.executeQuery(selectQuery);
-						        Long nrOfRows = JdbcCommon.convertToAvroStream(resultSet, out);
-								logger.info("Result FlowFile contain {} Avro records", new Object[]{nrOfRows});
-								
-							} catch (SQLException e) {
-								throw new ProcessException(e);						
-							}
-						}
-					});
-	
-					logger.info("Transferred {} to 'success'", new Object[]{outgoing});
-					session.getProvenanceReporter().modifyContent(outgoing, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
-					session.transfer(outgoing, REL_SUCCESS);
-				} finally {
-					st.close();
-				}
-			} finally {
-				// return connection to pool
-				con.close();
-			}
-
-		} catch (ProcessException e) {
-			logger.error("Unable to execute sql select query due to {}", new Object[]{e});
-			session.transfer(incoming, REL_FAILURE);
-		} catch (SQLException e) {
-			logger.error("Unable to execute sql select query due to {}", new Object[]{e});
+
+        try (final Connection con = dbcpService.getConnection();
+            final Statement st = con.createStatement()) {
+            st.setQueryTimeout(queryTimeout); // timeout in seconds
+            final LongHolder nrOfRows = new LongHolder(0L);
+            FlowFile outgoing = session.write(incoming, new OutputStreamCallback() {
+                @Override
+                public void process(final OutputStream out) throws IOException {
+                    try {
+                        logger.debug("Executing query {}", new Object[] { selectQuery });
+                        final ResultSet resultSet = st.executeQuery(selectQuery);
+                        nrOfRows.set(JdbcCommon.convertToAvroStream(resultSet, out));
+                    } catch (final SQLException e) {
+                        throw new ProcessException(e);
+                    }
+                }
+            });
+
+            logger.info("{} contains {} Avro records", new Object[] { nrOfRows.get() });
+            logger.info("Transferred {} to 'success'", new Object[] { outgoing });
+            session.getProvenanceReporter().modifyContent(outgoing, "Retrieved " + nrOfRows.get() + " rows", stopWatch.getElapsed(TimeUnit.MILLISECONDS));
+            session.transfer(outgoing, REL_SUCCESS);
+        } catch (final ProcessException | SQLException e) {
+            logger.error("Unable to execute SQL select query {} for {} due to {}; routing to failure", new Object[] { selectQuery, incoming, e });
 			session.transfer(incoming, REL_FAILURE);
 		}
     }

http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3b8e44a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
index c9ad423..8dff244 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
@@ -21,6 +21,7 @@ import java.io.OutputStream;
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
+import static java.sql.Types.*;
 
 import org.apache.avro.Schema;
 import org.apache.avro.SchemaBuilder;
@@ -34,104 +35,102 @@ import org.apache.avro.io.DatumWriter;
 
 /**
  *  JDBC / SQL common functions.
- *
  */
 public class JdbcCommon {
 
-	public static long convertToAvroStream(ResultSet rs, OutputStream outStream) throws SQLException, IOException {
-		
-		Schema schema = createSchema(rs);
-		GenericRecord rec = new GenericData.Record(schema);
-		
-		DatumWriter<GenericRecord> datumWriter  	= new GenericDatumWriter<GenericRecord>(schema);
-		DataFileWriter<GenericRecord> dataFileWriter= new DataFileWriter<GenericRecord>(datumWriter); 
-		dataFileWriter.create(schema, outStream);
-		
-		ResultSetMetaData meta = rs.getMetaData();
-		int nrOfColumns = meta.getColumnCount();
-		long nrOfRows = 0;
-		while (rs.next()) {
-			for (int i = 1; i <= nrOfColumns; i++) {
-				Object value = rs.getObject(i);
-				rec.put(i-1, value);
-			}
-			dataFileWriter.append(rec);
-			nrOfRows += 1;
+    public static long convertToAvroStream(final ResultSet rs, final OutputStream outStream) throws SQLException, IOException {
+        final Schema schema = createSchema(rs);
+        final GenericRecord rec = new GenericData.Record(schema);
+
+        final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema);
+        try (final DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter)) {
+            dataFileWriter.create(schema, outStream);
+
+            final ResultSetMetaData meta = rs.getMetaData();
+            final int nrOfColumns = meta.getColumnCount();
+            long nrOfRows = 0;
+            while (rs.next()) {
+                for (int i = 1; i <= nrOfColumns; i++) {
+                    final Object value = rs.getObject(i);
+                    rec.put(i - 1, value);
+                }
+                dataFileWriter.append(rec);
+                nrOfRows += 1;
+            }
+
+            return nrOfRows;
 		}
-
-		dataFileWriter.close();
-		return nrOfRows;
 	}
-	
-	public static Schema createSchema(ResultSet rs) throws SQLException {
-		
-		ResultSetMetaData meta = rs.getMetaData();
-		int nrOfColumns = meta.getColumnCount();
-		String tableName = meta.getTableName(1);
-		
-		FieldAssembler<Schema> builder = SchemaBuilder.record(tableName).namespace("any.data").fields();
-		
-		/**
-		 * 	Some missing Avro types - Decimal, Date types.
-		 * May need some additional work.
-		 */
-		for (int i = 1; i <= nrOfColumns; i++)
+
+    public static Schema createSchema(final ResultSet rs) throws SQLException {
+        final ResultSetMetaData meta = rs.getMetaData();
+        final int nrOfColumns = meta.getColumnCount();
+        final String tableName = meta.getTableName(1);
+
+        final FieldAssembler<Schema> builder = SchemaBuilder.record(tableName).namespace("any.data").fields();
+
+        /**
+         * Some missing Avro types - Decimal, Date types. May need some
+         * additional work.
+         */
+        for (int i = 1; i <= nrOfColumns; i++) {
 			switch (meta.getColumnType(i)) {
-			
-			case java.sql.Types.CHAR:
-			case java.sql.Types.LONGNVARCHAR:
-			case java.sql.Types.LONGVARCHAR:
-			case java.sql.Types.NCHAR:
-			case java.sql.Types.NVARCHAR:
-			case java.sql.Types.VARCHAR:
-				builder.name(meta.getColumnName(i)).type().stringType().noDefault();			
-				break;
-
-			case java.sql.Types.BOOLEAN:
-				builder.name(meta.getColumnName(i)).type().booleanType().noDefault();			
-				break;
-
-			case java.sql.Types.INTEGER:
-			case java.sql.Types.SMALLINT:
-			case java.sql.Types.TINYINT:
-				builder.name(meta.getColumnName(i)).type().intType().noDefault();			
-				break;
-
-			case java.sql.Types.BIGINT:
-				builder.name(meta.getColumnName(i)).type().longType().noDefault();			
-				break;
-
-			// java.sql.RowId is interface, is seems to be database implementation specific, let's convert to String
-			case java.sql.Types.ROWID:
-				builder.name(meta.getColumnName(i)).type().stringType().noDefault();			
-				break;
-
-			case java.sql.Types.FLOAT:
-			case java.sql.Types.REAL:
-				builder.name(meta.getColumnName(i)).type().floatType().noDefault();			
-				break;
-
-			case java.sql.Types.DOUBLE:
-				builder.name(meta.getColumnName(i)).type().doubleType().noDefault();			
-				break;
-
-			// TODO Did not find direct suitable type, need to be clarified!!!!
-			case java.sql.Types.DECIMAL:
-			case java.sql.Types.NUMERIC:
-				builder.name(meta.getColumnName(i)).type().stringType().noDefault();			
-				break;
-
-			// TODO Did not find direct suitable type, need to be clarified!!!!
-			case java.sql.Types.DATE:
-			case java.sql.Types.TIME:
-			case java.sql.Types.TIMESTAMP:
-				builder.name(meta.getColumnName(i)).type().stringType().noDefault();			
-				break;
-
-			default:
-				break;
+                case CHAR:
+                case LONGNVARCHAR:
+                case LONGVARCHAR:
+                case NCHAR:
+                case NVARCHAR:
+                case VARCHAR:
+                    builder.name(meta.getColumnName(i)).type().stringType().noDefault();
+                    break;
+
+                case BOOLEAN:
+                    builder.name(meta.getColumnName(i)).type().booleanType().noDefault();
+                    break;
+
+                case INTEGER:
+                case SMALLINT:
+                case TINYINT:
+                    builder.name(meta.getColumnName(i)).type().intType().noDefault();
+                    break;
+
+                case BIGINT:
+                    builder.name(meta.getColumnName(i)).type().longType().noDefault();
+                    break;
+
+                // java.sql.RowId is interface, is seems to be database
+                // implementation specific, let's convert to String
+                case ROWID:
+                    builder.name(meta.getColumnName(i)).type().stringType().noDefault();
+                    break;
+
+                case FLOAT:
+                case REAL:
+                    builder.name(meta.getColumnName(i)).type().floatType().noDefault();
+                    break;
+
+                case DOUBLE:
+                    builder.name(meta.getColumnName(i)).type().doubleType().noDefault();
+                    break;
+
+                // Did not find direct suitable type, need to be clarified!!!!
+                case DECIMAL:
+                case NUMERIC:
+                    builder.name(meta.getColumnName(i)).type().stringType().noDefault();
+                    break;
+
+                // Did not find direct suitable type, need to be clarified!!!!
+                case DATE:
+                case TIME:
+                case TIMESTAMP:
+                    builder.name(meta.getColumnName(i)).type().stringType().noDefault();
+                    break;
+
+                default:
+                    break;
 			}
+        }
+
 		return builder.endRecord();
 	}
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3b8e44a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java
index b9243ef..30da519 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java
@@ -65,7 +65,7 @@ public class TestExecuteSQL {
     public static void setup() {
         System.setProperty("derby.stream.error.file", "target/derby.log");
     }
-    
+
     @Test
     public void testNoTimeLimit() throws InitializationException, ClassNotFoundException, SQLException, IOException {
     	invokeOnTrigger(null);
@@ -77,44 +77,45 @@ public class TestExecuteSQL {
     	invokeOnTrigger(1);		// 1 second max time
     }
 
-    public void invokeOnTrigger(Integer queryTimeout) throws InitializationException, ClassNotFoundException, SQLException, IOException {
+    public void invokeOnTrigger(final Integer queryTimeout) throws InitializationException, ClassNotFoundException, SQLException, IOException {
         final TestRunner runner = TestRunners.newTestRunner(ExecuteSQL.class);
-        
+
         final DBCPService dbcp = new DBCPServiceSimpleImpl();
         final Map<String, String> dbcpProperties = new HashMap<>();
 
         runner.addControllerService("dbcp", dbcp, dbcpProperties);
-        
+
         runner.enableControllerService(dbcp);
         runner.setProperty(ExecuteSQL.DBCP_SERVICE, "dbcp");
-        
-        if (queryTimeout!=null)
+
+        if (queryTimeout != null) {
         	runner.setProperty(ExecuteSQL.QUERY_TIMEOUT, queryTimeout.toString() + " secs");
-        
+        }
+
         // remove previous test database, if any
-        File dbLocation = new File(DB_LOCATION);
+        final File dbLocation = new File(DB_LOCATION);
         dbLocation.delete();
 
         // load test data to database
-        Connection con = dbcp.getConnection();
+        final Connection con = dbcp.getConnection();
         TestJdbcHugeStream.loadTestData2Database(con, 100, 2000, 1000);
-        System.out.println("test data loaded");
-        
+        LOGGER.info("test data loaded");
+
         // ResultSet size will be 1x2000x1000 = 2 000 000 rows
         // because of where PER.ID = ${person.id}
         final int nrOfRows = 2000000;
-        String query = "select "
+        final String query = "select "
         		+ "  PER.ID as PersonId, PER.NAME as PersonName, PER.CODE as PersonCode"
         		+ ", PRD.ID as ProductId,PRD.NAME as ProductName,PRD.CODE as ProductCode"
         		+ ", REL.ID as RelId,    REL.NAME as RelName,    REL.CODE as RelCode"
         		+ ", ROW_NUMBER() OVER () as rownr "
         		+ " from persons PER, products PRD, relationships REL"
         		+ " where PER.ID = ${person.id}";
-        
+
         runner.setProperty(ExecuteSQL.SQL_SELECT_QUERY, query);
 
         // incoming FlowFile content is not used, but attributes are used
-        Map<String,String> attributes = new HashMap<String,String>();
+        final Map<String,String> attributes = new HashMap<String,String>();
         attributes.put("person.id", "10");
         runner.enqueue("Hello".getBytes(), attributes);
 
@@ -122,27 +123,27 @@ public class TestExecuteSQL {
         runner.assertAllFlowFilesTransferred(ExecuteSQL.REL_SUCCESS, 1);
 
         // read all Avro records and verify created FlowFile contains 1000000 records
-        List<MockFlowFile> flowfiles = runner.getFlowFilesForRelationship(ExecuteSQL.REL_SUCCESS);
-        InputStream in = new ByteArrayInputStream(flowfiles.get(0).toByteArray());
-        DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
-        DataFileStream<GenericRecord> dataFileReader = new DataFileStream<GenericRecord>(in, datumReader);
+        final List<MockFlowFile> flowfiles = runner.getFlowFilesForRelationship(ExecuteSQL.REL_SUCCESS);
+        final InputStream in = new ByteArrayInputStream(flowfiles.get(0).toByteArray());
+        final DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
+        final DataFileStream<GenericRecord> dataFileReader = new DataFileStream<GenericRecord>(in, datumReader);
         GenericRecord record = null;
         long recordsFromStream = 0;
         while (dataFileReader.hasNext()) {
         	// Reuse record object by passing it to next(). This saves us from
         	// allocating and garbage collecting many objects for files with many items.
         	record = dataFileReader.next(record);
-//   	      	System.out.println(record);
           	recordsFromStream += 1;
         }
-        System.out.println("total nr of records from stream: " + recordsFromStream);
+
+        LOGGER.info("total nr of records from stream: " + recordsFromStream);
         assertEquals(nrOfRows, recordsFromStream);
         dataFileReader.close();
     }
 
-    
-    
-    
+
+
+
     /**
      * Simple implementation only for ExecuteSQL processor testing.
      *
@@ -157,13 +158,13 @@ public class TestExecuteSQL {
 		@Override
 		public Connection getConnection() throws ProcessException {
 	        try {
-				Class.forName("org.apache.derby.jdbc.EmbeddedDriver");        
-				Connection con = DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";create=true");
+				Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+				final Connection con = DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";create=true");
 				return con;
-			} catch (Exception e) {
+			} catch (final Exception e) {
 				throw new ProcessException("getConnection failed: " + e);
 			}
-		}    	
+		}
     }
-    
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3b8e44a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcHugeStream.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcHugeStream.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcHugeStream.java
index 8c54bc0..89c2fb3 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcHugeStream.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcHugeStream.java
@@ -39,17 +39,18 @@ import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.io.DatumReader;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
  *  Test streaming using large number of result set rows.
  * 1. Read data from database.
  * 2. Create Avro schema from ResultSet meta data.
- * 3. Read rows from ResultSet and write rows to Avro writer stream 
+ * 3. Read rows from ResultSet and write rows to Avro writer stream
  *    (Avro will create record for each row).
- * 4. And finally read records from Avro stream to verify all data is present in Avro stream. 
- *   
- *  
+ * 4. And finally read records from Avro stream to verify all data is present in Avro stream.
+ *
+ *
  * Sql query will return all combinations from 3 table.
  * For example when each table contain 1000 rows, result set will be 1 000 000 000 rows.
  *
@@ -63,7 +64,7 @@ public class TestJdbcHugeStream {
         System.setProperty("derby.stream.error.file", "target/derby.log");
     }
 
-    /**	
+    /**
      * 	In case of large record set this will fail with
      * java.lang.OutOfMemoryError: Java heap space
 	 * at java.util.Arrays.copyOf(Arrays.java:2271)
@@ -71,149 +72,147 @@ public class TestJdbcHugeStream {
 	 * at java.io.ByteArrayOutputStream.ensureCapacity(ByteArrayOutputStream.java:93)
 	 * at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:140)
 	 * at org.apache.avro.file.DataFileWriter$BufferedFileOutputStream$PositionFilter.write(DataFileWriter.java:446)
-     * 
-     */    
-//	@Test
+     *
+     */
+    @Test
+    @Ignore
 	public void readSend2StreamHuge_InMemory() throws ClassNotFoundException, SQLException, IOException {
-		
+
         // remove previous test database, if any
-        File dbLocation = new File(DB_LOCATION);
+        final File dbLocation = new File(DB_LOCATION);
         dbLocation.delete();
 
-        Connection con = createConnection();
-        loadTestData2Database(con, 150, 150, 150);
-        System.out.println("test data loaded");
-        
-        Statement st = con.createStatement();
-        
-        // Notice!
-        // Following select is deliberately invalid!
-        // For testing we need huge amount of rows, so where part is not used.
-        ResultSet resultSet = st.executeQuery("select "
-        		+ "  PER.ID as PersonId, PER.NAME as PersonName, PER.CODE as PersonCode"
-        		+ ", PRD.ID as ProductId,PRD.NAME as ProductName,PRD.CODE as ProductCode"
-        		+ ", REL.ID as RelId,    REL.NAME as RelName,    REL.CODE as RelCode"
-        		+ ", ROW_NUMBER() OVER () as rownr "
-        		+ " from persons PER, products PRD, relationships REL");
-
-        ByteArrayOutputStream outStream = new ByteArrayOutputStream();
-        long nrOfRows = JdbcCommon.convertToAvroStream(resultSet, outStream);
-        System.out.println("total nr of rows in resultset: " + nrOfRows);
-
-        byte[] serializedBytes = outStream.toByteArray();
-        assertNotNull(serializedBytes);
-        System.out.println("Avro serialized result size in bytes: " + serializedBytes.length);
-
-        // Deserialize bytes to records
-        
-        InputStream instream = new ByteArrayInputStream(serializedBytes);
-        
-        DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
-        DataFileStream<GenericRecord> dataFileReader = new DataFileStream<GenericRecord>(instream, datumReader);
-        GenericRecord record = null;
-        long recordsFromStream = 0;
-        while (dataFileReader.hasNext()) {
-        	// Reuse record object by passing it to next(). This saves us from
-        	// allocating and garbage collecting many objects for files with many items.
-        	record = dataFileReader.next(record);
-//        	System.out.println(record);
-        	recordsFromStream += 1;
+        try (final Connection con = createConnection()) {
+            loadTestData2Database(con, 150, 150, 150);
+            System.out.println("test data loaded");
+
+            try (final Statement st = con.createStatement()) {
+                // Notice!
+                // Following select is deliberately invalid!
+                // For testing we need huge amount of rows, so where part is not
+                // used.
+                final ResultSet resultSet = st.executeQuery("select "
+                    + "  PER.ID as PersonId, PER.NAME as PersonName, PER.CODE as PersonCode"
+                    + ", PRD.ID as ProductId,PRD.NAME as ProductName,PRD.CODE as ProductCode"
+                    + ", REL.ID as RelId,    REL.NAME as RelName,    REL.CODE as RelCode"
+                    + ", ROW_NUMBER() OVER () as rownr "
+                    + " from persons PER, products PRD, relationships REL");
+
+                final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+                final long nrOfRows = JdbcCommon.convertToAvroStream(resultSet, outStream);
+                System.out.println("total nr of rows in resultset: " + nrOfRows);
+
+                final byte[] serializedBytes = outStream.toByteArray();
+                assertNotNull(serializedBytes);
+                System.out.println("Avro serialized result size in bytes: " + serializedBytes.length);
+
+                // Deserialize bytes to records
+
+                final InputStream instream = new ByteArrayInputStream(serializedBytes);
+
+                final DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
+                try (final DataFileStream<GenericRecord> dataFileReader = new DataFileStream<GenericRecord>(instream, datumReader)) {
+                    GenericRecord record = null;
+                    long recordsFromStream = 0;
+                    while (dataFileReader.hasNext()) {
+                        // Reuse record object by passing it to next(). This
+                        // saves us from
+                        // allocating and garbage collecting many objects for
+                        // files with many items.
+                        record = dataFileReader.next(record);
+                        recordsFromStream += 1;
+                    }
+                    System.out.println("total nr of records from stream: " + recordsFromStream);
+                    assertEquals(nrOfRows, recordsFromStream);
+                }
+            }
         }
-        System.out.println("total nr of records from stream: " + recordsFromStream);
-        assertEquals(nrOfRows, recordsFromStream);
-        st.close();
-        con.close();
 	}
-		
+
 	@Test
 	public void readSend2StreamHuge_FileBased() throws ClassNotFoundException, SQLException, IOException {
-		
+
         // remove previous test database, if any
-        File dbLocation = new File(DB_LOCATION);
+        final File dbLocation = new File(DB_LOCATION);
         dbLocation.delete();
 
-        Connection con = createConnection();
+        try (final Connection con = createConnection()) {
         loadTestData2Database(con, 300, 300, 300);
-        System.out.println("test data loaded");
-        
-        Statement st = con.createStatement();
-        
-        // Notice!
-        // Following select is deliberately invalid!
-        // For testing we need huge amount of rows, so where part is not used.
-        ResultSet resultSet = st.executeQuery("select "
-        		+ "  PER.ID as PersonId, PER.NAME as PersonName, PER.CODE as PersonCode"
-        		+ ", PRD.ID as ProductId,PRD.NAME as ProductName,PRD.CODE as ProductCode"
-        		+ ", REL.ID as RelId,    REL.NAME as RelName,    REL.CODE as RelCode"
-        		+ ", ROW_NUMBER() OVER () as rownr "
-        		+ " from persons PER, products PRD, relationships REL");
-
-        OutputStream outStream = new FileOutputStream("target/data.avro");
-        long nrOfRows = JdbcCommon.convertToAvroStream(resultSet, outStream);
-        System.out.println("total nr of rows in resultset: " + nrOfRows);
-/*
-        byte[] serializedBytes = outStream.toByteArray();
-        assertNotNull(serializedBytes);
-        System.out.println("Avro serialized result size in bytes: " + serializedBytes.length);
-*/
-        // Deserialize bytes to records
-        
-        InputStream instream = new FileInputStream("target/data.avro");
-        
-        DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
-        DataFileStream<GenericRecord> dataFileReader = new DataFileStream<GenericRecord>(instream, datumReader);
-        GenericRecord record = null;
-        long recordsFromStream = 0;
-        while (dataFileReader.hasNext()) {
-        	// Reuse record object by passing it to next(). This saves us from
-        	// allocating and garbage collecting many objects for files with many items.
-        	record = dataFileReader.next(record);
-//        	System.out.println(record);
-        	recordsFromStream += 1;
+
+            try (final Statement st = con.createStatement()) {
+                // Notice!
+                // Following select is deliberately invalid!
+                // For testing we need huge amount of rows, so where part is not
+                // used.
+                final ResultSet resultSet = st.executeQuery("select "
+                    + "  PER.ID as PersonId, PER.NAME as PersonName, PER.CODE as PersonCode"
+                    + ", PRD.ID as ProductId,PRD.NAME as ProductName,PRD.CODE as ProductCode"
+                    + ", REL.ID as RelId,    REL.NAME as RelName,    REL.CODE as RelCode"
+                    + ", ROW_NUMBER() OVER () as rownr "
+                    + " from persons PER, products PRD, relationships REL");
+
+                final OutputStream outStream = new FileOutputStream("target/data.avro");
+                final long nrOfRows = JdbcCommon.convertToAvroStream(resultSet, outStream);
+
+                // Deserialize bytes to records
+                final InputStream instream = new FileInputStream("target/data.avro");
+
+                final DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
+                try (final DataFileStream<GenericRecord> dataFileReader = new DataFileStream<GenericRecord>(instream, datumReader)) {
+                    GenericRecord record = null;
+                    long recordsFromStream = 0;
+                    while (dataFileReader.hasNext()) {
+                        // Reuse record object by passing it to next(). This
+                        // saves us from
+                        // allocating and garbage collecting many objects for
+                        // files with many items.
+                        record = dataFileReader.next(record);
+                        recordsFromStream += 1;
+                    }
+                    System.out.println("total nr of records from stream: " + recordsFromStream);
+                    assertEquals(nrOfRows, recordsFromStream);
+                }
+            }
         }
-        System.out.println("total nr of records from stream: " + recordsFromStream);
-        assertEquals(nrOfRows, recordsFromStream);        
-        st.close();
-        con.close();
 	}
-		
+
 	//================================================  helpers  ===============================================
-	
-    static String dropPersons		= "drop table persons";
-    static String dropProducts		= "drop table products";
-    static String dropRelationships= "drop table relationships";
-    static String createPersons 		= "create table persons		(id integer, name varchar(100), code integer)";
-    static String createProducts 		= "create table products	(id integer, name varchar(100), code integer)";
-    static String createRelationships 	= "create table relationships(id integer,name varchar(100), code integer)";
+
+    static String dropPersons = "drop table persons";
+    static String dropProducts = "drop table products";
+    static String dropRelationships = "drop table relationships";
+    static String createPersons = "create table persons (id integer, name varchar(100), code integer)";
+    static String createProducts = "create table products (id integer, name varchar(100), code integer)";
+    static String createRelationships = "create table relationships (id integer,name varchar(100), code integer)";
 
 	static public void loadTestData2Database(Connection con, int nrOfPersons, int nrOfProducts, int nrOfRels) throws ClassNotFoundException, SQLException {
-		
+
 		System.out.println(createRandomName());
 		System.out.println(createRandomName());
 		System.out.println(createRandomName());
-		
-        Statement st = con.createStatement();
+
+        final Statement st = con.createStatement();
 
         // tables may not exist, this is not serious problem.
         try { st.executeUpdate(dropPersons);
-        } catch (Exception e) { }
-        
+        } catch (final Exception e) { }
+
         try { st.executeUpdate(dropProducts);
-        } catch (Exception e) { }
-        
+        } catch (final Exception e) { }
+
         try { st.executeUpdate(dropRelationships);
-        } catch (Exception e) { } 
+        } catch (final Exception e) { }
 
         st.executeUpdate(createPersons);
         st.executeUpdate(createProducts);
         st.executeUpdate(createRelationships);
-        
+
         for (int i = 0; i < nrOfPersons; i++)
      		loadPersons(st, i);
-		
+
         for (int i = 0; i < nrOfProducts; i++)
         	loadProducts(st, i);
-		
+
         for (int i = 0; i < nrOfRels; i++)
         	loadRelationships(st, i);
 
@@ -223,41 +222,37 @@ public class TestJdbcHugeStream {
 	static Random rng = new Random(53495);
 
 	static private void loadPersons(Statement st, int nr) throws SQLException {
-		
-        st.executeUpdate("insert into persons values (" + nr + ", '" + createRandomName() +  "', " + rng.nextInt(469946) + ")" );		
+        st.executeUpdate("insert into persons values (" + nr + ", '" + createRandomName() +  "', " + rng.nextInt(469946) + ")" );
 	}
 
 	static private void loadProducts(Statement st, int nr) throws SQLException {
-		
-        st.executeUpdate("insert into products values (" + nr + ", '" + createRandomName() +  "', " + rng.nextInt(469946) + ")" );		
+        st.executeUpdate("insert into products values (" + nr + ", '" + createRandomName() +  "', " + rng.nextInt(469946) + ")" );
 	}
 
 	static private void loadRelationships(Statement st, int nr) throws SQLException {
-		
-        st.executeUpdate("insert into relationships values (" + nr + ", '" + createRandomName() +  "', " + rng.nextInt(469946) + ")" );		
+        st.executeUpdate("insert into relationships values (" + nr + ", '" + createRandomName() +  "', " + rng.nextInt(469946) + ")" );
 	}
 
 	static private String createRandomName() {
 		return createRandomString() + " " + createRandomString();
 	}
-	
+
 	static private String createRandomString() {
-		
-		int length = rng.nextInt(19);
-		String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-		
-		char[] text = new char[length];
+
+		final int length = rng.nextInt(19);
+		final String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+		final char[] text = new char[length];
 	    for (int i = 0; i < length; i++)
 	    {
 	        text[i] = characters.charAt(rng.nextInt(characters.length()));
 	    }
-	    return new String(text);			
+	    return new String(text);
 	}
-	
+
 	private Connection createConnection() throws ClassNotFoundException, SQLException {
-		
-        Class.forName("org.apache.derby.jdbc.EmbeddedDriver");        
-        Connection con = DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";create=true");
+        Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+        final Connection con = DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";create=true");
 		return con;
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3b8e44a/nifi/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/pom.xml b/nifi/pom.xml
index 039057e..0c71ba8 100644
--- a/nifi/pom.xml
+++ b/nifi/pom.xml
@@ -857,6 +857,11 @@
                 <artifactId>json-path</artifactId>
                 <version>2.0.0</version>
             </dependency>
+            <dependency>
+			    <groupId>org.apache.derby</groupId>
+			    <artifactId>derby</artifactId>
+			    <version>10.11.1.1</version>
+		    </dependency>
         </dependencies>
     </dependencyManagement>
     <dependencies>