You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2014/12/22 12:59:13 UTC
[01/31] incubator-nifi git commit: NIFI-150: Initial import of
unfinished User Guide
Repository: incubator-nifi
Updated Branches:
refs/heads/NIFI-182 5e6de841d -> 24372962a
NIFI-150: Initial import of unfinished User Guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/578974e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/578974e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/578974e5
Branch: refs/heads/NIFI-182
Commit: 578974e5589148af4d533a90c1e0f106c175d6e0
Parents: 713312d
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Dec 17 14:17:54 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Dec 17 14:17:54 2014 -0500
----------------------------------------------------------------------
.../src/main/resources/docs/NiFi User Guide.odt | Bin 0 -> 372498 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/578974e5/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt b/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt
new file mode 100644
index 0000000..08d0f11
Binary files /dev/null and b/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt differ
[15/31] incubator-nifi git commit: NIFI-150 fixed error with
inserting license header
Posted by mc...@apache.org.
NIFI-150 fixed error with inserting license header
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/d8507687
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/d8507687
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/d8507687
Branch: refs/heads/NIFI-182
Commit: d850768765c5012d09134a042eed2d71beb28c45
Parents: 4031bd5
Author: joewitt <jo...@apache.org>
Authored: Fri Dec 19 15:32:55 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Fri Dec 19 15:32:55 2014 -0500
----------------------------------------------------------------------
nifi-docs/pom.xml | 57 ++++++++++++++++++++++++++++++++++----------------
1 file changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8507687/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
index bfa9e44..979f1f8 100644
--- a/nifi-docs/pom.xml
+++ b/nifi-docs/pom.xml
@@ -13,6 +13,27 @@
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-asciidoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <resources>
+ <resource>
+ <directory>src/main/asciidoc</directory>
+ </resource>
+ </resources>
+ <outputDirectory>${project.build.directory}/asciidoc</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.asciidoctor</groupId>
<artifactId>asciidoctor-maven-plugin</artifactId>
@@ -20,13 +41,14 @@
<executions>
<execution>
<id>output-html</id>
- <phase>generate-resources</phase>
+ <phase>prepare-package</phase>
<goals>
<goal>process-asciidoc</goal>
</goals>
</execution>
</executions>
<configuration>
+ <sourceDirectory>${project.build.directory}/asciidoc</sourceDirectory>
<backend>html</backend>
<attributes>
<skipFrontMatter>true</skipFrontMatter>
@@ -42,15 +64,14 @@
<version>1.5.3</version>
<executions>
<execution>
- <phase>prepare-package</phase>
+ <phase>package</phase>
<goals>
<goal>replace</goal>
</goals>
</execution>
</executions>
<configuration>
- <ignoreErrors>true</ignoreErrors>
- <file>target/generated-docs/nifi-user-guide.html</file>
+ <file>${project.build.directory}/generated-docs/nifi-user-guide.html</file>
<regex>true</regex>
<regexFlags>
<regexFlag>DOTALL</regexFlag>
@@ -59,22 +80,22 @@
<token>^(.*)$</token>
<value>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- $1
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+$1
</value>
</configuration>
</plugin>
[31/31] incubator-nifi git commit: Merge branch 'develop' into
NIFI-182
Posted by mc...@apache.org.
Merge branch 'develop' into NIFI-182
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/24372962
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/24372962
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/24372962
Branch: refs/heads/NIFI-182
Commit: 24372962aa823498cf740e44f784ddd90477f428
Parents: 5e6de84 0d8da14
Author: Matt Gilman <ma...@gmail.com>
Authored: Mon Dec 22 06:58:39 2014 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Dec 22 06:58:39 2014 -0500
----------------------------------------------------------------------
LICENSE | 384 +++++++++
NOTICE | 7 +
README.md | 32 +-
assembly/pom.xml | 199 ++---
assembly/src/main/assembly/dependencies.xml | 34 +-
commons/core-flowfile-attributes/pom.xml | 29 -
.../flowfile/attributes/CoreAttributes.java | 72 --
.../attributes/FlowFileAttributeKey.java | 21 -
commons/data-provenance-utils/pom.xml | 2 +-
commons/naive-search-ring-buffer/pom.xml | 30 -
.../apache/nifi/util/NaiveSearchRingBuffer.java | 135 ----
.../nifi/util/TestNaiveSearchRingBuffer.java | 72 --
commons/nifi-file-utils/pom.xml | 35 -
.../java/org/apache/nifi/file/FileUtils.java | 612 ---------------
commons/nifi-stream-utils/.gitignore | 1 -
commons/nifi-stream-utils/pom.xml | 30 -
.../org/apache/nifi/io/BufferedInputStream.java | 37 -
.../apache/nifi/io/BufferedOutputStream.java | 140 ----
.../apache/nifi/io/ByteArrayInputStream.java | 250 ------
.../apache/nifi/io/ByteArrayOutputStream.java | 250 ------
.../apache/nifi/io/ByteCountingInputStream.java | 104 ---
.../nifi/io/ByteCountingOutputStream.java | 63 --
.../org/apache/nifi/io/DataOutputStream.java | 417 ----------
.../org/apache/nifi/io/GZIPOutputStream.java | 41 -
.../nifi/io/LeakyBucketStreamThrottler.java | 324 --------
.../apache/nifi/io/NonCloseableInputStream.java | 56 --
.../nifi/io/NonCloseableOutputStream.java | 51 --
.../org/apache/nifi/io/NullOutputStream.java | 46 --
.../org/apache/nifi/io/StreamThrottler.java | 33 -
.../java/org/apache/nifi/io/StreamUtils.java | 257 ------
.../org/apache/nifi/io/ZipOutputStream.java | 38 -
.../exception/BytePatternNotFoundException.java | 28 -
.../io/util/NonThreadSafeCircularBuffer.java | 69 --
.../nifi/io/TestLeakyBucketThrottler.java | 144 ----
.../src/test/resources/logback-test.xml | 48 --
commons/nifi-utils/pom.xml | 4 +
.../flowfile/attributes/CoreAttributes.java | 72 ++
.../attributes/FlowFileAttributeKey.java | 21 +
.../apache/nifi/io/CompoundUpdateMonitor.java | 115 ---
.../org/apache/nifi/io/LastModifiedMonitor.java | 30 -
.../java/org/apache/nifi/io/MD5SumMonitor.java | 49 --
.../apache/nifi/io/SynchronousFileWatcher.java | 123 ---
.../java/org/apache/nifi/io/UpdateMonitor.java | 25 -
.../nifi/remote/StandardVersionNegotiator.java | 81 ++
.../apache/nifi/remote/VersionNegotiator.java | 65 ++
.../TransmissionDisabledException.java | 25 +
.../nifi/remote/io/CompressionInputStream.java | 184 +++++
.../nifi/remote/io/CompressionOutputStream.java | 147 ++++
.../remote/io/InterruptableInputStream.java | 117 +++
.../remote/io/InterruptableOutputStream.java | 81 ++
.../remote/io/socket/BufferStateManager.java | 111 +++
.../io/socket/SocketChannelInputStream.java | 157 ++++
.../io/socket/SocketChannelOutputStream.java | 113 +++
.../remote/io/socket/ssl/SSLSocketChannel.java | 602 ++++++++++++++
.../socket/ssl/SSLSocketChannelInputStream.java | 62 ++
.../ssl/SSLSocketChannelOutputStream.java | 53 ++
.../nifi/stream/io/BufferedInputStream.java | 37 +
.../nifi/stream/io/BufferedOutputStream.java | 140 ++++
.../nifi/stream/io/ByteArrayInputStream.java | 250 ++++++
.../nifi/stream/io/ByteArrayOutputStream.java | 250 ++++++
.../nifi/stream/io/ByteCountingInputStream.java | 104 +++
.../stream/io/ByteCountingOutputStream.java | 63 ++
.../apache/nifi/stream/io/DataOutputStream.java | 417 ++++++++++
.../apache/nifi/stream/io/GZIPOutputStream.java | 41 +
.../stream/io/LeakyBucketStreamThrottler.java | 324 ++++++++
.../nifi/stream/io/NonCloseableInputStream.java | 56 ++
.../stream/io/NonCloseableOutputStream.java | 51 ++
.../apache/nifi/stream/io/NullOutputStream.java | 46 ++
.../apache/nifi/stream/io/StreamThrottler.java | 33 +
.../org/apache/nifi/stream/io/StreamUtils.java | 257 ++++++
.../apache/nifi/stream/io/ZipOutputStream.java | 38 +
.../exception/BytePatternNotFoundException.java | 28 +
.../io/util/NonThreadSafeCircularBuffer.java | 69 ++
.../apache/nifi/util/NaiveSearchRingBuffer.java | 135 ++++
.../org/apache/nifi/util/file/FileUtils.java | 623 +++++++++++++++
.../file/monitor/CompoundUpdateMonitor.java | 115 +++
.../util/file/monitor/LastModifiedMonitor.java | 30 +
.../nifi/util/file/monitor/MD5SumMonitor.java | 51 ++
.../file/monitor/SynchronousFileWatcher.java | 123 +++
.../nifi/util/file/monitor/UpdateMonitor.java | 25 +
.../org/apache/nifi/util/search/Search.java | 57 ++
.../org/apache/nifi/util/search/SearchTerm.java | 141 ++++
.../util/search/ahocorasick/AhoCorasick.java | 155 ++++
.../nifi/util/search/ahocorasick/Node.java | 72 ++
.../util/search/ahocorasick/SearchState.java | 63 ++
.../io/TestCompressionInputOutputStreams.java | 153 ++++
.../stream/io/TestLeakyBucketThrottler.java | 147 ++++
.../nifi/util/TestNaiveSearchRingBuffer.java | 72 ++
.../file/monitor/TestCompoundUpdateMonitor.java | 71 ++
.../monitor/TestSynchronousFileWatcher.java | 61 ++
.../timebuffer/TestCompoundUpdateMonitor.java | 75 --
.../timebuffer/TestSynchronousFileWatcher.java | 64 --
.../src/test/resources/logback-test.xml | 32 +
commons/pom.xml | 6 -
commons/remote-communications-utils/pom.xml | 29 -
.../nifi/remote/StandardVersionNegotiator.java | 81 --
.../apache/nifi/remote/VersionNegotiator.java | 65 --
.../TransmissionDisabledException.java | 25 -
.../nifi/remote/io/CompressionInputStream.java | 184 -----
.../nifi/remote/io/CompressionOutputStream.java | 147 ----
.../remote/io/InterruptableInputStream.java | 117 ---
.../remote/io/InterruptableOutputStream.java | 81 --
.../remote/io/socket/BufferStateManager.java | 111 ---
.../io/socket/SocketChannelInputStream.java | 157 ----
.../io/socket/SocketChannelOutputStream.java | 113 ---
.../remote/io/socket/ssl/SSLSocketChannel.java | 602 --------------
.../socket/ssl/SSLSocketChannelInputStream.java | 62 --
.../ssl/SSLSocketChannelOutputStream.java | 53 --
.../io/TestCompressionInputOutputStreams.java | 153 ----
commons/search-utils/pom.xml | 30 -
.../org/apache/nifi/util/search/Search.java | 57 --
.../org/apache/nifi/util/search/SearchTerm.java | 141 ----
.../util/search/ahocorasick/AhoCorasick.java | 155 ----
.../nifi/util/search/ahocorasick/Node.java | 72 --
.../util/search/ahocorasick/SearchState.java | 63 --
commons/wali/pom.xml | 2 +-
.../org/wali/MinimalLockingWriteAheadLog.java | 4 +-
.../framework-bundle/framework/cluster/pom.xml | 2 +-
.../impl/FileBasedClusterNodeFirewall.java | 2 +-
.../nifi/cluster/flow/impl/DataFlowDaoImpl.java | 10 +-
.../impl/FileBasedClusterNodeFirewallTest.java | 3 +-
.../framework-bundle/framework/core-api/pom.xml | 6 +-
.../framework-bundle/framework/core/pom.xml | 8 -
.../nifi/controller/FileSystemSwapManager.java | 2 +-
.../apache/nifi/controller/FlowController.java | 2 +-
.../nifi/controller/FlowUnmarshaller.java | 2 +-
.../apache/nifi/controller/SnippetManager.java | 8 +-
.../nifi/controller/StandardFlowService.java | 2 +-
.../controller/StandardFlowSynchronizer.java | 2 +-
.../nifi/controller/StandardProcessorNode.java | 2 +-
.../apache/nifi/controller/TemplateManager.java | 8 +-
.../repository/FileSystemRepository.java | 4 +-
.../repository/StandardProcessSession.java | 6 +-
.../repository/VolatileContentRepository.java | 4 +-
.../service/ControllerServiceLoader.java | 2 +-
.../StandardXMLFlowConfigurationDAO.java | 2 +-
.../nifi/remote/StandardRemoteProcessGroup.java | 2 +-
.../TestWriteAheadFlowFileRepository.java | 2 +-
.../repository/io/TestLimitedOutputStream.java | 3 +-
.../file-authorization-provider/pom.xml | 2 +-
.../FileAuthorizationProvider.java | 2 +-
.../FileAuthorizationProviderTest.java | 2 +-
.../resources/src/main/resources/DISCLAIMER | 15 -
.../resources/src/main/resources/LICENSE | 202 -----
.../resources/src/main/resources/NOTICE | 7 -
.../src/main/resources/conf/bootstrap.conf | 22 +-
.../resources/src/main/resources/docs/README.md | 65 --
.../framework/site-to-site/pom.xml | 12 -
.../remote/codec/StandardFlowFileCodec.java | 2 +-
.../remote/io/socket/SocketChannelInput.java | 4 +-
.../remote/io/socket/SocketChannelOutput.java | 4 +-
.../io/socket/ssl/SSLSocketChannelInput.java | 4 +-
.../io/socket/ssl/SSLSocketChannelOutput.java | 4 +-
.../apache/nifi/web/api/ProvenanceResource.java | 2 +-
.../nifi/web/util/ClientResponseUtils.java | 2 +-
.../src/main/webapp/js/application.js | 5 +
.../hadoop-bundle/hdfs-processors/pom.xml | 6 +-
...lowFileStreamUnpackerSequenceFileWriter.java | 2 +-
.../apache/nifi/processors/hadoop/PutHDFS.java | 4 +-
.../hadoop/SequenceFileWriterImpl.java | 2 +-
.../hadoop/TarUnpackerSequenceFileWriter.java | 2 +-
.../hadoop/ZipUnpackerSequenceFileWriter.java | 2 +-
.../hadoop/util/OutputStreamWritable.java | 4 +-
.../monitor-threshold-bundle/processor/pom.xml | 8 -
.../processors/monitor/MonitorThreshold.java | 2 +-
nar-bundles/monitor-threshold-bundle/ui/pom.xml | 2 +-
.../persistent-provenance-repository/pom.xml | 8 -
.../nifi/provenance/StandardRecordReader.java | 4 +-
.../nifi/provenance/StandardRecordWriter.java | 6 +-
.../provenance/rollover/CompressionAction.java | 4 +-
.../provenance/serialization/RecordReaders.java | 2 +-
.../standard-bundle/jms-processors/pom.xml | 6 +-
.../apache/nifi/processors/jms/JmsConsumer.java | 2 +-
.../org/apache/nifi/processors/jms/PutJMS.java | 2 +-
.../nifi/processors/jms/util/JmsFactory.java | 30 +-
.../nifi/processors/jms/util/JmsProperties.java | 3 +-
.../index.html | 1 -
.../index.html | 1 -
.../index.html | 1 -
.../standard-bundle/standard-processors/pom.xml | 14 +-
.../processors/standard/CompressContent.java | 6 +-
.../processors/standard/EncryptContent.java | 2 +-
.../standard/EvaluateRegularExpression.java | 2 +-
.../nifi/processors/standard/EvaluateXPath.java | 4 +-
.../processors/standard/EvaluateXQuery.java | 4 +-
.../standard/ExecuteStreamCommand.java | 6 +-
.../nifi/processors/standard/HashContent.java | 4 +-
.../nifi/processors/standard/ListenHTTP.java | 4 +-
.../nifi/processors/standard/MergeContent.java | 8 +-
.../nifi/processors/standard/ModifyBytes.java | 2 +-
.../nifi/processors/standard/PostHTTP.java | 12 +-
.../processors/standard/PutFileTransfer.java | 2 +-
.../nifi/processors/standard/ReplaceText.java | 2 +-
.../standard/ReplaceTextWithMapping.java | 2 +-
.../processors/standard/RouteOnContent.java | 2 +-
.../nifi/processors/standard/ScanAttribute.java | 4 +-
.../nifi/processors/standard/ScanContent.java | 6 +-
.../nifi/processors/standard/SplitContent.java | 2 +-
.../nifi/processors/standard/SplitText.java | 8 +-
.../nifi/processors/standard/SplitXml.java | 2 +-
.../nifi/processors/standard/TransformXml.java | 2 +-
.../nifi/processors/standard/UnpackContent.java | 6 +-
.../standard/servlets/ListenHTTPServlet.java | 4 +-
.../standard/util/UDPStreamConsumer.java | 2 +-
.../processors/standard/TestMergeContent.java | 3 +-
.../processors/standard/TestScanContent.java | 4 +-
.../distributed-cache-client-service/pom.xml | 6 +-
.../DistributedMapCacheClientService.java | 4 +-
.../DistributedSetCacheClientService.java | 4 +-
.../cache/client/SSLCommsSession.java | 4 +-
.../cache/client/StandardCommsSession.java | 4 +-
.../distributed-cache-protocol/pom.xml | 2 +-
.../distributed-cache-server/pom.xml | 6 +-
.../cache/server/AbstractCacheServer.java | 4 +-
.../cache/server/SetCacheServer.java | 2 +-
.../cache/server/map/MapCacheServer.java | 2 +-
.../update-attribute-bundle/processor/pom.xml | 2 +-
nifi-docs/pom.xml | 124 +++
.../images/add-processor-with-tag-cloud.png | Bin 0 -> 22925 bytes
.../src/main/asciidoc/images/add-processor.png | Bin 0 -> 31524 bytes
.../src/main/asciidoc/images/addConnect.png | Bin 0 -> 1996 bytes
.../src/main/asciidoc/images/comments-tab.png | Bin 0 -> 6431 bytes
.../src/main/asciidoc/images/components.png | Bin 0 -> 8992 bytes
.../asciidoc/images/connection-settings.png | Bin 0 -> 15214 bytes
.../main/asciidoc/images/create-connection.png | Bin 0 -> 9430 bytes
.../asciidoc/images/edit-property-dropdown.png | Bin 0 -> 14291 bytes
.../asciidoc/images/edit-property-textarea.png | Bin 0 -> 17146 bytes
.../src/main/asciidoc/images/iconAlert.png | Bin 0 -> 1396 bytes
.../src/main/asciidoc/images/iconConnection.png | Bin 0 -> 1517 bytes
.../src/main/asciidoc/images/iconDelete.png | Bin 0 -> 670 bytes
.../src/main/asciidoc/images/iconDisable.png | Bin 0 -> 764 bytes
nifi-docs/src/main/asciidoc/images/iconEdit.png | Bin 0 -> 493 bytes
.../src/main/asciidoc/images/iconEnable.png | Bin 0 -> 667 bytes
.../src/main/asciidoc/images/iconFunnel.png | Bin 0 -> 1223 bytes
nifi-docs/src/main/asciidoc/images/iconInfo.png | Bin 0 -> 550 bytes
.../src/main/asciidoc/images/iconInputPort.png | Bin 0 -> 1842 bytes
.../src/main/asciidoc/images/iconLabel.png | Bin 0 -> 838 bytes
.../src/main/asciidoc/images/iconNotSecure.png | Bin 0 -> 137 bytes
.../src/main/asciidoc/images/iconOutputPort.png | Bin 0 -> 1658 bytes
.../main/asciidoc/images/iconProcessGroup.png | Bin 0 -> 1422 bytes
.../src/main/asciidoc/images/iconProcessor.png | Bin 0 -> 1446 bytes
.../asciidoc/images/iconRemoteProcessGroup.png | Bin 0 -> 674 bytes
nifi-docs/src/main/asciidoc/images/iconRun.png | Bin 0 -> 538 bytes
.../src/main/asciidoc/images/iconSecure.png | Bin 0 -> 133 bytes
nifi-docs/src/main/asciidoc/images/iconStop.png | Bin 0 -> 402 bytes
.../src/main/asciidoc/images/iconTemplate.png | Bin 0 -> 970 bytes
.../asciidoc/images/iconTransmissionActive.png | Bin 0 -> 1330 bytes
.../images/iconTransmissionInactive.png | Bin 0 -> 1248 bytes
.../images/instantiate-template-description.png | Bin 0 -> 6508 bytes
.../asciidoc/images/instantiate-template.png | Bin 0 -> 3216 bytes
.../main/asciidoc/images/invalid-processor.png | Bin 0 -> 8187 bytes
nifi-docs/src/main/asciidoc/images/new-flow.png | Bin 0 -> 53989 bytes
.../main/asciidoc/images/nifi-navigation.png | Bin 0 -> 85458 bytes
.../asciidoc/images/nifi-toolbar-components.png | Bin 0 -> 72263 bytes
.../main/asciidoc/images/processor-anatomy.png | Bin 0 -> 37035 bytes
.../images/processor-connection-bubble.png | Bin 0 -> 9936 bytes
.../src/main/asciidoc/images/properties-tab.png | Bin 0 -> 11847 bytes
.../images/remote-group-ports-dialog.png | Bin 0 -> 21662 bytes
.../images/remote-port-connection-status.png | Bin 0 -> 57332 bytes
.../src/main/asciidoc/images/scheduling-tab.png | Bin 0 -> 12288 bytes
.../src/main/asciidoc/images/settings-tab.png | Bin 0 -> 16225 bytes
.../src/main/asciidoc/images/status-bar.png | Bin 0 -> 57171 bytes
.../main/asciidoc/images/valid-processor.png | Bin 0 -> 8090 bytes
.../src/main/asciidoc/nifi-user-guide.adoc | 782 +++++++++++++++++++
nifi-docs/src/main/assembly/dependencies.xml | 28 +
nifi-mock/pom.xml | 4 -
pom.xml | 56 +-
267 files changed, 7893 insertions(+), 7111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/24372962/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
----------------------------------------------------------------------
diff --cc nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
index 5e5ee1c,e9b0499..748fc62
--- a/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
+++ b/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
@@@ -255,12 -255,12 +255,17 @@@ $(document).ready(function ()
selectComponent(componentName);
}
});
+
+ // listen for on the rest api and user guide
+ $('a.rest-api a.user-guide').on('click', function() {
+ selectComponent($(this).text());
+ });
+ // listen for on the rest api and user guide
+ $('a.rest-api a.user-guide').on('click', function() {
+ selectComponent($(this).text());
+ });
+
// get the initial selection
var initialComponentLink = $('a.component-link:first');
var initialSelection = $('#initial-selection').text();
[28/31] incubator-nifi git commit: NIFI-192 fixed rpm build,
assembly is cleaned up, and references to the license, notice, disclaimer,
etc.. are to those at the root only
Posted by mc...@apache.org.
NIFI-192 fixed rpm build, assembly is cleaned up, and references to the license, notice, disclaimer, etc.. are to those at the root only
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/a3664a00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/a3664a00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/a3664a00
Branch: refs/heads/NIFI-182
Commit: a3664a00c8e2ed60b4c159041f8ae3cf108e1840
Parents: 1da5a9c
Author: joewitt <jo...@apache.org>
Authored: Sat Dec 20 20:42:40 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sat Dec 20 20:42:40 2014 -0500
----------------------------------------------------------------------
README.md | 2 +-
assembly/pom.xml | 170 +++++++---------
assembly/src/main/assembly/dependencies.xml | 27 +--
.../resources/src/main/resources/DISCLAIMER | 15 --
.../resources/src/main/resources/LICENSE | 202 -------------------
.../resources/src/main/resources/NOTICE | 7 -
.../resources/src/main/resources/README | 84 --------
.../resources/src/main/resources/docs/README.md | 65 ------
8 files changed, 88 insertions(+), 484 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index bc4c008..5fe3cb4 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Apache NiFi supports powerful and scalable directed graphs of data routing, tran
## Getting Started
-Execute <nifi install dir>/bin/nifi.sh
+Execute <nifi install dir>/bin/nifi.sh start
## Getting Help
If you have questions, you can reach out to our mailing list: dev@nifi.incubator.apache.org
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 887086c..45ac535 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -345,16 +345,10 @@
<nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay>
<nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads>
<nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration>
- <!--
- Properties to execute nifi using Maven. It is assumed that the assembly has a
- format of type 'dir' so that the executable can be referenced in a defined location.
- -->
- <nifi.assembly.id>bin</nifi.assembly.id>
- <nifi.executable>${project.artifactId}-${project.version}/bin/${project.artifactId}</nifi.executable>
</properties>
<profiles>
<profile>
- <id>standard-rpm</id>
+ <id>rpm</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
@@ -363,101 +357,91 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>rpm</goal>
- </goals>
- </execution>
- </executions>
<configuration>
- <name>${project.artifactId}</name>
- <projversion>${project.version}</projversion>
<summary>Apache NiFi (incubating)</summary>
- <description>
- Apache Nifi (incubating) is dataflow system
- based on the Flow-Based Programming concepts.
- </description>
- <copyright>NONE</copyright>
- <url>http://nifi.apache.org</url>
+ <description>Apache Nifi (incubating) is dataflow system based on the Flow-Based Programming concepts.</description>
+ <license>Apache License, Version 2.0 and others (see included LICENSE file)</license>
+ <url>http://nifi.incubator.apache.org</url>
<group>Utilities</group>
- <requires>
- <require>jdk</require>
- </requires>
- <prefix>/opt/${project.artifactId}</prefix>
+ <prefix>/opt/nifi</prefix>
<defineStatements>
<defineStatement>_use_internal_dependency_generator 0</defineStatement>
</defineStatements>
<defaultDirmode>750</defaultDirmode>
<defaultFilemode>640</defaultFilemode>
- <defaultUsername>nifi</defaultUsername>
- <defaultGroupname>nifi</defaultGroupname>
- <mappings>
- <mapping>
- <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}</directory>
- </mapping>
- <mapping>
- <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/bin</directory>
- <filemode>750</filemode>
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/bin/nifi</location>
- <destination>${project.artifactId}</destination>
- <filter>true</filter>
- </source>
- <source>
- <location>${project.build.directory}/generated-resources/bin/wrapper-linux-x86-32</location>
- </source>
- <source>
- <location>${project.build.directory}/generated-resources/bin/wrapper-linux-x86-64</location>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/conf</directory>
- <configuration>true</configuration>
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/conf</location>
- <excludes>
- <exclude>nifi.properties</exclude>
- </excludes>
- <filter>true</filter>
- </source>
- <source>
- <location>${project.build.directory}/generated-resources/conf/nifi.properties</location>
- <destination>${project.artifactId}.properties</destination>
- <filter>true</filter>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/lib</directory>
- <dependency />
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/lib</location>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/docs</directory>
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/docs</location>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/logs</directory>
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/logs</location>
- </source>
- </sources>
- </mapping>
- </mappings>
+ <defaultUsername>root</defaultUsername>
+ <defaultGroupname>root</defaultGroupname>
</configuration>
+ <executions>
+ <execution>
+ <id>build-bin-rpm</id>
+ <goals>
+ <goal>attached-rpm</goal>
+ </goals>
+ <configuration>
+ <classifier>bin</classifier>
+ <provides>
+ <provide>nifi</provide>
+ </provides>
+ <requires>
+ <require>jdk</require>
+ </requires>
+ <mappings>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}</directory>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}</directory>
+ <sources>
+ <source>
+ <location>../LICENSE</location>
+ </source>
+ <source>
+ <location>../NOTICE</location>
+ </source>
+ <source>
+ <location>../README.md</location>
+ <destination>README</destination>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/bin</directory>
+ <filemode>750</filemode>
+ <sources>
+ <source>
+ <location>${project.build.directory}/generated-resources/bin/nifi.sh</location>
+ <destination>nifi.sh</destination>
+ <filter>true</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/conf</directory>
+ <configuration>true</configuration>
+ <sources>
+ <source>
+ <location>${project.build.directory}/generated-resources/conf</location>
+ <filter>true</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/lib</directory>
+ <dependency />
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/docs</directory>
+ <sources>
+ <source>
+ <location>${project.build.directory}/generated-docs</location>
+ </source>
+ </sources>
+ </mapping>
+ </mappings>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/assembly/src/main/assembly/dependencies.xml
----------------------------------------------------------------------
diff --git a/assembly/src/main/assembly/dependencies.xml b/assembly/src/main/assembly/dependencies.xml
index 13e5b83..4b77a32 100644
--- a/assembly/src/main/assembly/dependencies.xml
+++ b/assembly/src/main/assembly/dependencies.xml
@@ -106,38 +106,31 @@
<filtered>true</filtered>
</file>
<file>
- <source>${project.build.directory}/generated-resources/docs/README.md</source>
- <outputDirectory>docs</outputDirectory>
- <destName>README.md</destName>
- <fileMode>0640</fileMode>
- <filtered>true</filtered>
- </file>
- <file>
- <source>${project.build.directory}/generated-resources/DISCLAIMER</source>
+ <source>../README.md</source>
<outputDirectory>./</outputDirectory>
- <destName>DISCLAIMER</destName>
- <fileMode>0640</fileMode>
+ <destName>README</destName>
+ <fileMode>0644</fileMode>
<filtered>true</filtered>
</file>
<file>
- <source>${project.build.directory}/generated-resources/DISCLAIMER</source>
+ <source>../DISCLAIMER</source>
<outputDirectory>./</outputDirectory>
<destName>DISCLAIMER</destName>
- <fileMode>0640</fileMode>
+ <fileMode>0644</fileMode>
<filtered>true</filtered>
- </file>
+ </file>
<file>
- <source>${project.build.directory}/generated-resources/LICENSE</source>
+ <source>../LICENSE</source>
<outputDirectory>./</outputDirectory>
<destName>LICENSE</destName>
- <fileMode>0640</fileMode>
+ <fileMode>0644</fileMode>
<filtered>true</filtered>
</file>
<file>
- <source>${project.build.directory}/generated-resources/NOTICE</source>
+ <source>../NOTICE</source>
<outputDirectory>./</outputDirectory>
<destName>NOTICE</destName>
- <fileMode>0640</fileMode>
+ <fileMode>0644</fileMode>
<filtered>true</filtered>
</file>
</files>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/nar-bundles/framework-bundle/framework/resources/src/main/resources/DISCLAIMER
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/DISCLAIMER b/nar-bundles/framework-bundle/framework/resources/src/main/resources/DISCLAIMER
deleted file mode 100644
index 0f8e7a1..0000000
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/DISCLAIMER
+++ /dev/null
@@ -1,15 +0,0 @@
-Apache NiFi is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further review
-indicates that the infrastructure, communications, and decision making process
-have stabilized in a manner consistent with other successful ASF projects.
-
-While incubation status is not necessarily a reflection of the completeness
-or stability of the code, it does indicate that the project has yet to be
-fully endorsed by the ASF.
-
-For more information about the incubation status of the Apache NiFi project
-you can go to the following page:
-
-http://nifi.incubator.apache.org/
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/nar-bundles/framework-bundle/framework/resources/src/main/resources/LICENSE
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/LICENSE b/nar-bundles/framework-bundle/framework/resources/src/main/resources/LICENSE
deleted file mode 100644
index d645695..0000000
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/nar-bundles/framework-bundle/framework/resources/src/main/resources/NOTICE
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/NOTICE b/nar-bundles/framework-bundle/framework/resources/src/main/resources/NOTICE
deleted file mode 100644
index e02ae44..0000000
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/NOTICE
+++ /dev/null
@@ -1,7 +0,0 @@
-Apache NiFi
-Copyright 2014 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-Please see LICENSE for additional copyright and licensing information.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/nar-bundles/framework-bundle/framework/resources/src/main/resources/README
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/README b/nar-bundles/framework-bundle/framework/resources/src/main/resources/README
deleted file mode 100644
index 97c4964..0000000
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/README
+++ /dev/null
@@ -1,84 +0,0 @@
-***********************************************
-Introduction
-
-Apache NiFi is a dataflow system based on the concepts of flow-based programming. It is currently apart of the Apache Incubator.
-
-***********************************************
-Features
-
-Apache NiFi supports powerful and scalable directed graphs of data routing, transformation, and system mediation logic. Some of the high-level capabilities and objectives of Apache NiFi include:
-
-- Web-based user interface for seamless experience between design, control, feedback, and monitoring of data flows
-- Highly configurable along several dimensions of quality of service such as loss tolerant versus guaranteed delivery, low latency versus high throughput, and priority based queuing
-- Fine-grained data provenance for all data received, forked, joined, cloned, modified, sent, and ultimately dropped as data reaches its configured end-state
-- Component-based extension model along well defined interfaces enabling rapid development and effective testing
-
-************************************************
-Getting Started
-
-Execute <nifi install dir>/bin/nifi.sh start
-
-*************************************************
-Getting Help
-
-If you have questions, you can reach out to our mailing list: dev@nifi.incubator.apache.org
-([archive](http://mail-archives.apache.org/mod_mbox/incubator-nifi-dev)).
-We're also often available in IRC: #nifi on
-[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi).
-
-*************************************************
-Requirements
-
-JDK 1.7 or higher
-
-*************************************************
-License and Notices
-
-See
- <nifi install dir>/LICENSE
- <nifi install dir>/NOTICE
-
-**************************************************
-Disclaimer
-
-Apache NiFi is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further review
-indicates that the infrastructure, communications, and decision making process
-have stabilized in a manner consistent with other successful ASF projects.
-
-While incubation status is not necessarily a reflection of the completeness
-or stability of the code, it does indicate that the project has yet to be
-fully endorsed by the ASF.
-
-****************************************************
-Export Control
-
-This distribution includes cryptographic software. The country in which you
-currently reside may have restrictions on the import, possession, use, and/or
-re-export to another country, of encryption software. BEFORE using any
-encryption software, please check your country's laws, regulations and
-policies concerning the import, possession, or use, and re-export of encryption
-software, to see if this is permitted. See <http://www.wassenaar.org/> for more
-information.
-
-The U.S. Government Department of Commerce, Bureau of Industry and Security
-(BIS), has classified this software as Export Commodity Control Number (ECCN)
-5D002.C.1, which includes information security software using or performing
-cryptographic functions with asymmetric algorithms. The form and manner of this
-Apache Software Foundation distribution makes it eligible for export under the
-License Exception ENC Technology Software Unrestricted (TSU) exception (see the
-BIS Export Administration Regulations, Section 740.13) for both object code and
-source code.
-
-The following provides more details on the included cryptographic software:
-
-Apache NiFi uses BouncyCastle, Jasypt, JCraft Inc., and the built-in
-java cryptography libraries for SSL, SSH, and the protection
-of sensitive configuration parameters. See
-http://bouncycastle.org/about.html
-http://www.jasypt.org/faq.html
-http://jcraft.com/c-info.html
-http://www.oracle.com/us/products/export/export-regulations-345813.html
-for more details on each of these libraries cryptography features.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a3664a00/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/README.md
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/README.md b/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/README.md
deleted file mode 100644
index 3f05134..0000000
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/README.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Apache NiFi
-
-Apache NiFi is a dataflow system based on the concepts of flow-based programming. It is currently apart of the Apache Incubator.
-
-## Table of Contents
-
-- [Features](#features)
-- [Getting Started](#getting-started)
-- [Getting Help](#getting-help)
-- [Requirements](#requirements)
-- [License](#license)
-- [Disclaimer](#disclaimer)
-
-## Features
-
-Apache NiFi supports powerful and scalable directed graphs of data routing, transformation, and system mediation logic. Some of the high-level capabilities and objectives of Apache NiFi include:
-
-- Web-based user interface for seamless experience between design, control, feedback, and monitoring of data flows
-- Highly configurable along several dimensions of quality of service such as loss tolerant versus guaranteed delivery, low latency versus high throughput, and priority based queuing
-- Fine-grained data provenance for all data received, forked, joined, cloned, modified, sent, and ultimately dropped as data reaches its configured end-state
-- Component-based extension model along well defined interfaces enabling rapid development and effective testing
-
-## Getting Started
-
-Execute <nifi install dir>/bin/nifi.sh
-
-## Getting Help
-If you have questions, you can reach out to our mailing list: dev@nifi.incubator.apache.org
-([archive](http://mail-archives.apache.org/mod_mbox/incubator-nifi-dev)).
-We're also often available in IRC: #nifi on
-[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi).
-
-
-## Requirements
-* JDK 1.7 or higher
-
-## License
-
-Except as otherwise noted this software is licensed under the
-[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html)
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-## Disclaimer
-
-Apache NiFi is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further review
-indicates that the infrastructure, communications, and decision making process
-have stabilized in a manner consistent with other successful ASF projects.
-
-While incubation status is not necessarily a reflection of the completeness
-or stability of the code, it does indicate that the project has yet to be
-fully endorsed by the ASF.
[23/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/io/UpdateMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/io/UpdateMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/io/UpdateMonitor.java
deleted file mode 100644
index 33fb444..0000000
--- a/commons/nifi-utils/src/main/java/org/apache/nifi/io/UpdateMonitor.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-public interface UpdateMonitor {
-
- Object getCurrentState(Path path) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
new file mode 100644
index 0000000..77c34c9
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class StandardVersionNegotiator implements VersionNegotiator {
+
+ private final List<Integer> versions;
+ private int curVersion;
+
+ public StandardVersionNegotiator(final int... supportedVersions) {
+ if (Objects.requireNonNull(supportedVersions).length == 0) {
+ throw new IllegalArgumentException("At least one version must be supported");
+ }
+
+ final List<Integer> supported = new ArrayList<>();
+ for (final int version : supportedVersions) {
+ supported.add(version);
+ }
+ this.versions = Collections.unmodifiableList(supported);
+ this.curVersion = supportedVersions[0];
+ }
+
+ @Override
+ public int getVersion() {
+ return curVersion;
+ }
+
+ @Override
+ public void setVersion(final int version) throws IllegalArgumentException {
+ if (!isVersionSupported(version)) {
+ throw new IllegalArgumentException("Version " + version + " is not supported");
+ }
+
+ this.curVersion = version;
+ }
+
+ @Override
+ public int getPreferredVersion() {
+ return versions.get(0);
+ }
+
+ @Override
+ public Integer getPreferredVersion(final int maxVersion) {
+ for (final Integer version : this.versions) {
+ if (maxVersion >= version) {
+ return version;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean isVersionSupported(final int version) {
+ return versions.contains(version);
+ }
+
+ @Override
+ public List<Integer> getSupportedVersions() {
+ return versions;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
new file mode 100644
index 0000000..74f9b3d
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.util.List;
+
+public interface VersionNegotiator {
+
+ /**
+ * @return the currently configured Version of this resource
+ */
+ int getVersion();
+
+ /**
+ * Sets the version of this resource to the specified version. Only the
+ * lower byte of the version is relevant.
+ *
+ * @param version
+ * @throws IllegalArgumentException if the given Version is not supported by
+ * this resource, as is indicated by the {@link #isVersionSupported(int)}
+ * method
+ */
+ void setVersion(int version) throws IllegalArgumentException;
+
+ /**
+ *
+ * @return the Version of this resource that is preferred
+ */
+ int getPreferredVersion();
+
+ /**
+ * Gets the preferred version of this resource that is no greater than the
+ * given maxVersion. If no acceptable version exists that is less than
+ * <code>maxVersion</code>, then <code>null</code> is returned
+ *
+ * @param maxVersion
+ * @return
+ */
+ Integer getPreferredVersion(int maxVersion);
+
+ /**
+ * Indicates whether or not the specified version is supported by this
+ * resource
+ *
+ * @param version
+ * @return
+ */
+ boolean isVersionSupported(int version);
+
+ List<Integer> getSupportedVersions();
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
new file mode 100644
index 0000000..05fd915
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.exception;
+
+/**
+ * Indicates that the user disabled transmission while communications were
+ * taking place with a peer
+ */
+public class TransmissionDisabledException extends RuntimeException {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
new file mode 100644
index 0000000..71cf894
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+public class CompressionInputStream extends InputStream {
+
+ private final InputStream in;
+ private final Inflater inflater;
+
+ private byte[] compressedBuffer;
+ private byte[] buffer;
+
+ private int bufferIndex;
+ private boolean eos = false; // whether or not we've reached the end of stream
+ private boolean allDataRead = false; // different from eos b/c eos means allDataRead == true && buffer is empty
+
+ private final byte[] fourByteBuffer = new byte[4];
+
+ public CompressionInputStream(final InputStream in) {
+ this.in = in;
+ inflater = new Inflater();
+
+ buffer = new byte[0];
+ compressedBuffer = new byte[0];
+ bufferIndex = 1;
+ }
+
+ private String toHex(final byte[] array) {
+ final StringBuilder sb = new StringBuilder("0x");
+ for (final byte b : array) {
+ final String hex = Integer.toHexString(b).toUpperCase();
+ if (hex.length() == 1) {
+ sb.append("0");
+ }
+ sb.append(hex);
+ }
+ return sb.toString();
+ }
+
+ protected void readChunkHeader() throws IOException {
+ // Ensure that we have a valid SYNC chunk
+ fillBuffer(fourByteBuffer);
+ if (!Arrays.equals(CompressionOutputStream.SYNC_BYTES, fourByteBuffer)) {
+ throw new IOException("Invalid CompressionInputStream. Expected first 4 bytes to be 'SYNC' but were " + toHex(fourByteBuffer));
+ }
+
+ // determine the size of the decompressed buffer
+ fillBuffer(fourByteBuffer);
+ buffer = new byte[toInt(fourByteBuffer)];
+
+ // determine the size of the compressed buffer
+ fillBuffer(fourByteBuffer);
+ compressedBuffer = new byte[toInt(fourByteBuffer)];
+
+ bufferIndex = buffer.length; // indicate that buffer is empty
+ }
+
+ private int toInt(final byte[] data) {
+ return ((data[0] & 0xFF) << 24)
+ | ((data[1] & 0xFF) << 16)
+ | ((data[2] & 0xFF) << 8)
+ | (data[3] & 0xFF);
+ }
+
+ protected void bufferAndDecompress() throws IOException {
+ if (allDataRead) {
+ eos = true;
+ return;
+ }
+
+ readChunkHeader();
+ fillBuffer(compressedBuffer);
+
+ inflater.setInput(compressedBuffer);
+ try {
+ inflater.inflate(buffer);
+ } catch (final DataFormatException e) {
+ throw new IOException(e);
+ }
+ inflater.reset();
+
+ bufferIndex = 0;
+ final int moreDataByte = in.read();
+ if (moreDataByte < 1) {
+ allDataRead = true;
+ } else if (moreDataByte > 1) {
+ throw new IOException("Expected indicator of whether or not more data was to come (-1, 0, or 1) but got " + moreDataByte);
+ }
+ }
+
+ private void fillBuffer(final byte[] buffer) throws IOException {
+ int len;
+ int bytesLeft = buffer.length;
+ int bytesRead = 0;
+ while (bytesLeft > 0 && (len = in.read(buffer, bytesRead, bytesLeft)) > 0) {
+ bytesLeft -= len;
+ bytesRead += len;
+ }
+
+ if (bytesRead < buffer.length) {
+ throw new EOFException();
+ }
+ }
+
+ private boolean isBufferEmpty() {
+ return bufferIndex >= buffer.length;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (eos) {
+ return -1;
+ }
+
+ if (isBufferEmpty()) {
+ bufferAndDecompress();
+ }
+
+ if (isBufferEmpty()) {
+ eos = true;
+ return -1;
+ }
+
+ return buffer[bufferIndex++];
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ if (eos) {
+ return -1;
+ }
+
+ if (isBufferEmpty()) {
+ bufferAndDecompress();
+ }
+
+ if (isBufferEmpty()) {
+ eos = true;
+ return -1;
+ }
+
+ final int free = buffer.length - bufferIndex;
+ final int bytesToTransfer = Math.min(len, free);
+ System.arraycopy(buffer, bufferIndex, b, off, bytesToTransfer);
+ bufferIndex += bytesToTransfer;
+
+ return bytesToTransfer;
+ }
+
+ /**
+ * Does nothing. Does NOT close underlying InputStream
+ * @throws java.io.IOException
+ */
+ @Override
+ public void close() throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
new file mode 100644
index 0000000..bc46b0f
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.Deflater;
+
+public class CompressionOutputStream extends OutputStream {
+
+ public static final byte[] SYNC_BYTES = new byte[]{'S', 'Y', 'N', 'C'};
+
+ public static final int DEFAULT_COMPRESSION_LEVEL = 1;
+ public static final int DEFAULT_BUFFER_SIZE = 64 << 10;
+ public static final int MIN_BUFFER_SIZE = 8 << 10;
+
+ private final OutputStream out;
+ private final Deflater deflater;
+
+ private final byte[] buffer;
+ private final byte[] compressed;
+
+ private int bufferIndex = 0;
+ private boolean dataWritten = false;
+
+ public CompressionOutputStream(final OutputStream outStream) {
+ this(outStream, DEFAULT_BUFFER_SIZE);
+ }
+
+ public CompressionOutputStream(final OutputStream outStream, final int bufferSize) {
+ this(outStream, bufferSize, DEFAULT_COMPRESSION_LEVEL, Deflater.DEFAULT_STRATEGY);
+ }
+
+ public CompressionOutputStream(final OutputStream outStream, final int bufferSize, final int level, final int strategy) {
+ if (bufferSize < MIN_BUFFER_SIZE) {
+ throw new IllegalArgumentException("Buffer size must be at least " + MIN_BUFFER_SIZE);
+ }
+
+ this.out = outStream;
+ this.deflater = new Deflater(level);
+ this.deflater.setStrategy(strategy);
+ buffer = new byte[bufferSize];
+ compressed = new byte[bufferSize + 64];
+ }
+
+ /**
+ * Compresses the currently buffered chunk of data and sends it to the
+ * output stream
+ *
+ * @throws IOException
+ */
+ protected void compressAndWrite() throws IOException {
+ if (bufferIndex <= 0) {
+ return;
+ }
+
+ deflater.setInput(buffer, 0, bufferIndex);
+ deflater.finish();
+ final int compressedBytes = deflater.deflate(compressed);
+
+ writeChunkHeader(compressedBytes);
+ out.write(compressed, 0, compressedBytes);
+
+ bufferIndex = 0;
+ deflater.reset();
+ }
+
+ private void writeChunkHeader(final int compressedBytes) throws IOException {
+ // If we have already written data, write out a '1' to indicate that we have more data; when we close
+ // the stream, we instead write a '0' to indicate that we are finished sending data.
+ if (dataWritten) {
+ out.write(1);
+ }
+ out.write(SYNC_BYTES);
+ dataWritten = true;
+
+ writeInt(out, bufferIndex);
+ writeInt(out, compressedBytes);
+ }
+
+ private void writeInt(final OutputStream out, final int val) throws IOException {
+ out.write(val >>> 24);
+ out.write(val >>> 16);
+ out.write(val >>> 8);
+ out.write(val);
+ }
+
+ protected boolean bufferFull() {
+ return bufferIndex >= buffer.length;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ buffer[bufferIndex++] = (byte) (b & 0xFF);
+ if (bufferFull()) {
+ compressAndWrite();
+ }
+ }
+
+ @Override
+ public void write(final byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(final byte[] b, final int off, final int len) throws IOException {
+ int bytesLeft = len;
+ while (bytesLeft > 0) {
+ final int free = buffer.length - bufferIndex;
+ final int bytesThisIteration = Math.min(bytesLeft, free);
+ System.arraycopy(b, off + len - bytesLeft, buffer, bufferIndex, bytesThisIteration);
+ bufferIndex += bytesThisIteration;
+
+ bytesLeft -= bytesThisIteration;
+ if (bufferFull()) {
+ compressAndWrite();
+ }
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ compressAndWrite();
+ super.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ compressAndWrite();
+ out.write(0); // indicate that the stream is finished.
+ out.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
new file mode 100644
index 0000000..e03dfbf
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+
+public class InterruptableInputStream extends InputStream {
+
+ private volatile boolean interrupted = false;
+ private final InputStream in;
+
+ public InterruptableInputStream(final InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.read();
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.read(b);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.read(b, off, len);
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ in.close();
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ in.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported() {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.markSupported();
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ in.reset();
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.skip(n);
+ }
+
+ public void interrupt() {
+ interrupted = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
new file mode 100644
index 0000000..cba5be6
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+
+public class InterruptableOutputStream extends OutputStream {
+
+ private final OutputStream out;
+ private volatile boolean interrupted = false;
+
+ public InterruptableOutputStream(final OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.close();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.flush();
+ }
+
+ public void interrupt() {
+ this.interrupted = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
new file mode 100644
index 0000000..68913bd
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.nio.ByteBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BufferStateManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(BufferStateManager.class);
+
+ private ByteBuffer buffer;
+ private Direction direction = Direction.WRITE;
+
+ public BufferStateManager(final ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ public BufferStateManager(final ByteBuffer buffer, final Direction direction) {
+ this.buffer = buffer;
+ this.direction = direction;
+ }
+
+ /**
+ * Ensures that the buffer is at least as big as the size specified,
+ * resizing the buffer if necessary. This operation MAY change the direction
+ * of the buffer.
+ *
+ * @param requiredSize
+ */
+ public void ensureSize(final int requiredSize) {
+ if (buffer.capacity() < requiredSize) {
+ final ByteBuffer newBuffer = ByteBuffer.allocate(requiredSize);
+
+ // we have to read buffer so make sure the direction is correct.
+ if (direction == Direction.WRITE) {
+ buffer.flip();
+ }
+
+ // Copy from buffer to newBuffer
+ newBuffer.put(buffer);
+
+ // Swap the buffers
+ buffer = newBuffer;
+
+ // the new buffer is ready to be written to
+ direction = Direction.WRITE;
+ }
+ }
+
+ public ByteBuffer prepareForWrite(final int requiredSize) {
+ ensureSize(requiredSize);
+
+ if (direction == Direction.READ) {
+ direction = Direction.WRITE;
+ buffer.position(buffer.limit());
+ }
+
+ buffer.limit(buffer.capacity());
+ return buffer;
+ }
+
+ public ByteBuffer prepareForRead(final int requiredSize) {
+ ensureSize(requiredSize);
+
+ if (direction == Direction.WRITE) {
+ direction = Direction.READ;
+ buffer.flip();
+ }
+
+ return buffer;
+ }
+
+ /**
+ * Clears the contents of the buffer and sets direction to WRITE
+ */
+ public void clear() {
+ logger.debug("Clearing {}", buffer);
+ buffer.clear();
+ direction = Direction.WRITE;
+ }
+
+ public void compact() {
+ final String before = buffer.toString();
+ buffer.compact();
+ logger.debug("Before compact: {}, after: {}", before, buffer);
+ direction = Direction.WRITE;
+ }
+
+ public static enum Direction {
+
+ READ, WRITE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
new file mode 100644
index 0000000..32a3f26
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+public class SocketChannelInputStream extends InputStream {
+
+ private static final long CHANNEL_EMPTY_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
+ private final SocketChannel channel;
+ private volatile int timeoutMillis = 30000;
+
+ private final ByteBuffer oneByteBuffer = ByteBuffer.allocate(1);
+ private Byte bufferedByte = null;
+
+ public SocketChannelInputStream(final SocketChannel socketChannel) throws IOException {
+ // this class expects a non-blocking channel
+ socketChannel.configureBlocking(false);
+ this.channel = socketChannel;
+ }
+
+ public void setTimeout(final int timeoutMillis) {
+ this.timeoutMillis = timeoutMillis;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (bufferedByte != null) {
+ final int retVal = bufferedByte & 0xFF;
+ bufferedByte = null;
+ return retVal;
+ }
+
+ oneByteBuffer.flip();
+ oneByteBuffer.clear();
+
+ final long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesRead;
+ do {
+ bytesRead = channel.read(oneByteBuffer);
+ if (bytesRead == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out reading from socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_EMPTY_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked read operation
+ }
+ }
+ } while (bytesRead == 0);
+
+ if (bytesRead == -1) {
+ return -1;
+ }
+ oneByteBuffer.flip();
+ return oneByteBuffer.get() & 0xFF;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ if (bufferedByte != null) {
+ final byte retVal = bufferedByte;
+ bufferedByte = null;
+ b[off] = retVal;
+ return 1;
+ }
+
+ final ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
+
+ final long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesRead;
+ do {
+ bytesRead = channel.read(buffer);
+ if (bytesRead == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out reading from socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_EMPTY_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked read operation
+ }
+ }
+ } while (bytesRead == 0);
+
+ return bytesRead;
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (bufferedByte != null) {
+ return 1;
+ }
+
+ isDataAvailable(); // attempt to read from socket
+ return (bufferedByte == null) ? 0 : 1;
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ if (bufferedByte != null) {
+ return true;
+ }
+
+ oneByteBuffer.flip();
+ oneByteBuffer.clear();
+ final int bytesRead = channel.read(oneByteBuffer);
+ if (bytesRead == -1) {
+ throw new EOFException("Peer has closed the stream");
+ }
+ if (bytesRead > 0) {
+ oneByteBuffer.flip();
+ bufferedByte = oneByteBuffer.get();
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Closes the underlying socket channel.
+ * @throws java.io.IOException
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
new file mode 100644
index 0000000..77049ad
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+public class SocketChannelOutputStream extends OutputStream {
+
+ private static final long CHANNEL_FULL_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
+ private final SocketChannel channel;
+ private volatile int timeout = 30000;
+
+ private final ByteBuffer oneByteBuffer = ByteBuffer.allocate(1);
+
+ public SocketChannelOutputStream(final SocketChannel socketChannel) throws IOException {
+ // this class expects a non-blocking channel
+ socketChannel.configureBlocking(false);
+ this.channel = socketChannel;
+ }
+
+ public void setTimeout(final int timeoutMillis) {
+ this.timeout = timeoutMillis;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ oneByteBuffer.flip();
+ oneByteBuffer.clear();
+ oneByteBuffer.put((byte) b);
+ oneByteBuffer.flip();
+
+ final int timeoutMillis = this.timeout;
+ long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesWritten;
+ while (oneByteBuffer.hasRemaining()) {
+ bytesWritten = channel.write(oneByteBuffer);
+ if (bytesWritten == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out writing to socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_FULL_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked write operation
+ }
+ } else {
+ return;
+ }
+ }
+ }
+
+ @Override
+ public void write(final byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(final byte[] b, final int off, final int len) throws IOException {
+ final ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
+
+ final int timeoutMillis = this.timeout;
+ long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesWritten;
+ while (buffer.hasRemaining()) {
+ bytesWritten = channel.write(buffer);
+ if (bytesWritten == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out writing to socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_FULL_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked write operation
+ }
+ } else {
+ maxTime = System.currentTimeMillis() + timeoutMillis;
+ }
+ }
+ }
+
+ /**
+ * Closes the underlying SocketChannel
+ * @throws java.io.IOException
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
new file mode 100644
index 0000000..5810488
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
@@ -0,0 +1,602 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLEngineResult.Status;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.security.cert.CertificateExpiredException;
+import javax.security.cert.CertificateNotYetValidException;
+import javax.security.cert.X509Certificate;
+
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+import org.apache.nifi.remote.io.socket.BufferStateManager;
+import org.apache.nifi.remote.io.socket.BufferStateManager.Direction;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSLSocketChannel implements Closeable {
+
+ public static final int MAX_WRITE_SIZE = 65536;
+
+ private static final Logger logger = LoggerFactory.getLogger(SSLSocketChannel.class);
+ private static final long BUFFER_FULL_EMPTY_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
+
+ private final String hostname;
+ private final int port;
+ private final SSLEngine engine;
+ private final SocketAddress socketAddress;
+
+ private BufferStateManager streamInManager;
+ private BufferStateManager streamOutManager;
+ private BufferStateManager appDataManager;
+
+ private SocketChannel channel;
+
+ private final byte[] oneByteBuffer = new byte[1];
+
+ private int timeoutMillis = 30000;
+ private volatile boolean connected = false;
+ private boolean handshaking = false;
+ private boolean closed = false;
+ private volatile boolean interrupted = false;
+
+ public SSLSocketChannel(final SSLContext sslContext, final String hostname, final int port, final boolean client) throws IOException {
+ this.socketAddress = new InetSocketAddress(hostname, port);
+ this.channel = SocketChannel.open();
+ this.hostname = hostname;
+ this.port = port;
+ this.engine = sslContext.createSSLEngine();
+ this.engine.setUseClientMode(client);
+ engine.setNeedClientAuth(true);
+
+ streamInManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ streamOutManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ appDataManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getApplicationBufferSize()));
+ }
+
+ public SSLSocketChannel(final SSLContext sslContext, final SocketChannel socketChannel, final boolean client) throws IOException {
+ if (!socketChannel.isConnected()) {
+ throw new IllegalArgumentException("Cannot pass an un-connected SocketChannel");
+ }
+
+ this.channel = socketChannel;
+
+ this.socketAddress = socketChannel.getRemoteAddress();
+ final Socket socket = socketChannel.socket();
+ this.hostname = socket.getInetAddress().getHostName();
+ this.port = socket.getPort();
+
+ this.engine = sslContext.createSSLEngine();
+ this.engine.setUseClientMode(client);
+ engine.setNeedClientAuth(true);
+
+ streamInManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ streamOutManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ appDataManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getApplicationBufferSize()));
+ }
+
+ public void setTimeout(final int millis) {
+ this.timeoutMillis = millis;
+ }
+
+ public int getTimeout() {
+ return timeoutMillis;
+ }
+
+ public void connect() throws SSLHandshakeException, IOException {
+ try {
+ channel.configureBlocking(false);
+ if (!channel.isConnected()) {
+ final long startTime = System.currentTimeMillis();
+
+ if (!channel.connect(socketAddress)) {
+ while (!channel.finishConnect()) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+ if (System.currentTimeMillis() > startTime + timeoutMillis) {
+ throw new SocketTimeoutException("Timed out connecting to " + hostname + ":" + port);
+ }
+
+ try {
+ Thread.sleep(50L);
+ } catch (final InterruptedException e) {
+ }
+ }
+ }
+ }
+ engine.beginHandshake();
+
+ performHandshake();
+ logger.debug("{} Successfully completed SSL handshake", this);
+
+ streamInManager.clear();
+ streamOutManager.clear();
+ appDataManager.clear();
+
+ connected = true;
+ } catch (final Exception e) {
+ logger.error("{} Failed to connect due to {}", this, e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ closeQuietly(channel);
+ engine.closeInbound();
+ engine.closeOutbound();
+ throw e;
+ }
+ }
+
+ public String getDn() throws CertificateExpiredException, CertificateNotYetValidException, SSLPeerUnverifiedException {
+ final X509Certificate[] certs = engine.getSession().getPeerCertificateChain();
+ if (certs == null || certs.length == 0) {
+ throw new SSLPeerUnverifiedException("No certificates found");
+ }
+
+ final X509Certificate cert = certs[0];
+ cert.checkValidity();
+ return cert.getSubjectDN().getName().trim();
+ }
+
+ private void performHandshake() throws IOException {
+ // Generate handshake message
+ final byte[] emptyMessage = new byte[0];
+ handshaking = true;
+ logger.debug("{} Performing Handshake", this);
+
+ try {
+ while (true) {
+ switch (engine.getHandshakeStatus()) {
+ case FINISHED:
+ return;
+ case NEED_WRAP: {
+ final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);
+
+ final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+
+ final SSLEngineResult wrapHelloResult = engine.wrap(appDataOut, outboundBuffer);
+ if (wrapHelloResult.getStatus() == Status.BUFFER_OVERFLOW) {
+ streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ continue;
+ }
+
+ if (wrapHelloResult.getStatus() != Status.OK) {
+ throw new SSLHandshakeException("Could not generate SSL Handshake information: SSLEngineResult: "
+ + wrapHelloResult.toString());
+ }
+
+ logger.trace("{} Handshake response after wrapping: {}", this, wrapHelloResult);
+
+ final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
+ final int bytesToSend = readableStreamOut.remaining();
+ writeFully(readableStreamOut);
+ logger.trace("{} Sent {} bytes of wrapped data for handshake", this, bytesToSend);
+
+ streamOutManager.clear();
+ }
+ continue;
+ case NEED_UNWRAP: {
+ final ByteBuffer readableDataIn = streamInManager.prepareForRead(0);
+ final ByteBuffer appData = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+
+ // Read handshake response from other side
+ logger.trace("{} Unwrapping: {} to {}", new Object[]{this, readableDataIn, appData});
+ SSLEngineResult handshakeResponseResult = engine.unwrap(readableDataIn, appData);
+ logger.trace("{} Handshake response after unwrapping: {}", this, handshakeResponseResult);
+
+ if (handshakeResponseResult.getStatus() == Status.BUFFER_UNDERFLOW) {
+ final ByteBuffer writableDataIn = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ final int bytesRead = readData(writableDataIn);
+ if (bytesRead > 0) {
+ logger.trace("{} Read {} bytes for handshake", this, bytesRead);
+ }
+
+ if (bytesRead < 0) {
+ throw new SSLHandshakeException("Reached End-of-File marker while performing handshake");
+ }
+ } else if (handshakeResponseResult.getStatus() == Status.CLOSED) {
+ throw new IOException("Channel was closed by peer during handshake");
+ } else {
+ streamInManager.compact();
+ appDataManager.clear();
+ }
+ }
+ break;
+ case NEED_TASK:
+ performTasks();
+ continue;
+ case NOT_HANDSHAKING:
+ return;
+ }
+ }
+ } finally {
+ handshaking = false;
+ }
+ }
+
+ private void performTasks() {
+ Runnable runnable;
+ while ((runnable = engine.getDelegatedTask()) != null) {
+ runnable.run();
+ }
+ }
+
+ private void closeQuietly(final Closeable closeable) {
+ try {
+ closeable.close();
+ } catch (final Exception e) {
+ }
+ }
+
+ private int readData(final ByteBuffer dest) throws IOException {
+ final long startTime = System.currentTimeMillis();
+
+ while (true) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ if (dest.remaining() == 0) {
+ return 0;
+ }
+
+ final int readCount = channel.read(dest);
+
+ if (readCount == 0) {
+ if (System.currentTimeMillis() > startTime + timeoutMillis) {
+ throw new SocketTimeoutException("Timed out reading from socket connected to " + hostname + ":" + port);
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(BUFFER_FULL_EMPTY_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException();
+ }
+
+ continue;
+ }
+
+ logger.trace("{} Read {} bytes", this, readCount);
+ return readCount;
+ }
+ }
+
+ private Status encryptAndWriteFully(final BufferStateManager src) throws IOException {
+ SSLEngineResult result = null;
+
+ final ByteBuffer buff = src.prepareForRead(0);
+ final ByteBuffer outBuff = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+
+ logger.trace("{} Encrypting {} bytes", this, buff.remaining());
+ while (buff.remaining() > 0) {
+ result = engine.wrap(buff, outBuff);
+ if (result.getStatus() == Status.OK) {
+ final ByteBuffer readableOutBuff = streamOutManager.prepareForRead(0);
+ writeFully(readableOutBuff);
+ streamOutManager.clear();
+ } else {
+ return result.getStatus();
+ }
+ }
+
+ return result.getStatus();
+ }
+
+ private void writeFully(final ByteBuffer src) throws IOException {
+ long lastByteWrittenTime = System.currentTimeMillis();
+
+ int bytesWritten = 0;
+ while (src.hasRemaining()) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ final int written = channel.write(src);
+ bytesWritten += written;
+ final long now = System.currentTimeMillis();
+ if (written > 0) {
+ lastByteWrittenTime = now;
+ } else {
+ if (now > lastByteWrittenTime + timeoutMillis) {
+ throw new SocketTimeoutException("Timed out writing to socket connected to " + hostname + ":" + port);
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(BUFFER_FULL_EMPTY_WAIT_NANOS);
+ } catch (final InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException();
+ }
+ }
+ }
+
+ logger.trace("{} Wrote {} bytes", this, bytesWritten);
+ }
+
+ public boolean isClosed() {
+ if (closed) {
+ return true;
+ }
+ // need to detect if peer has sent closure handshake...if so the answer is true
+ final ByteBuffer writableInBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ int readCount = 0;
+ try {
+ readCount = channel.read(writableInBuffer);
+ } catch (IOException e) {
+ logger.error("{} Failed to readData due to {}", new Object[]{this, e});
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ readCount = -1; // treat the condition same as if End of Stream
+ }
+ if (readCount == 0) {
+ return false;
+ }
+ if (readCount > 0) {
+ logger.trace("{} Read {} bytes", this, readCount);
+
+ final ByteBuffer streamInBuffer = streamInManager.prepareForRead(1);
+ final ByteBuffer appDataBuffer = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ try {
+ SSLEngineResult unwrapResponse = engine.unwrap(streamInBuffer, appDataBuffer);
+ logger.trace("{} When checking if closed, (handshake={}) Unwrap response: {}", new Object[]{this, handshaking, unwrapResponse});
+ if (unwrapResponse.getStatus().equals(Status.CLOSED)) {
+ // Drain the incoming TCP buffer
+ final ByteBuffer discardBuffer = ByteBuffer.allocate(8192);
+ int bytesDiscarded = channel.read(discardBuffer);
+ while (bytesDiscarded > 0) {
+ discardBuffer.clear();
+ bytesDiscarded = channel.read(discardBuffer);
+ }
+ engine.closeInbound();
+ } else {
+ streamInManager.compact();
+ return false;
+ }
+ } catch (IOException e) {
+ logger.error("{} Failed to check if closed due to {}. Closing channel.", new Object[]{this, e});
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+ // either readCount is -1, indicating an end of stream, or the peer sent a closure handshake
+ // so go ahead and close down the channel
+ closeQuietly(channel.socket());
+ closeQuietly(channel);
+ closed = true;
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {
+ logger.debug("{} Closing Connection", this);
+ if (channel == null) {
+ return;
+ }
+
+ if (closed) {
+ return;
+ }
+
+ try {
+ engine.closeOutbound();
+
+ final byte[] emptyMessage = new byte[0];
+
+ final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);
+ final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ final SSLEngineResult handshakeResult = engine.wrap(appDataOut, outboundBuffer);
+
+ if (handshakeResult.getStatus() != Status.CLOSED) {
+ throw new IOException("Invalid close state - will not send network data");
+ }
+
+ final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
+ writeFully(readableStreamOut);
+ } finally {
+ // Drain the incoming TCP buffer
+ final ByteBuffer discardBuffer = ByteBuffer.allocate(8192);
+ try {
+ int bytesDiscarded = channel.read(discardBuffer);
+ while (bytesDiscarded > 0) {
+ discardBuffer.clear();
+ bytesDiscarded = channel.read(discardBuffer);
+ }
+ } catch (Exception e) {
+ }
+
+ closeQuietly(channel.socket());
+ closeQuietly(channel);
+ closed = true;
+ }
+ }
+
+ private int copyFromAppDataBuffer(final byte[] buffer, final int offset, final int len) {
+ // If any data already exists in the application data buffer, copy it to the buffer.
+ final ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
+
+ final int appDataRemaining = appDataBuffer.remaining();
+ if (appDataRemaining > 0) {
+ final int bytesToCopy = Math.min(len, appDataBuffer.remaining());
+ appDataBuffer.get(buffer, offset, bytesToCopy);
+
+ final int bytesCopied = appDataRemaining - appDataBuffer.remaining();
+ logger.trace("{} Copied {} ({}) bytes from unencrypted application buffer to user space",
+ new Object[]{this, bytesToCopy, bytesCopied});
+ return bytesCopied;
+ }
+ return 0;
+ }
+
+ public int available() throws IOException {
+ ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
+ ByteBuffer streamDataBuffer = streamInManager.prepareForRead(1);
+ final int buffered = appDataBuffer.remaining() + streamDataBuffer.remaining();
+ if (buffered > 0) {
+ return buffered;
+ }
+
+ final boolean wasAbleToRead = isDataAvailable();
+ if (!wasAbleToRead) {
+ return 0;
+ }
+
+ appDataBuffer = appDataManager.prepareForRead(1);
+ streamDataBuffer = streamInManager.prepareForRead(1);
+ return appDataBuffer.remaining() + streamDataBuffer.remaining();
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ final ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
+ final ByteBuffer streamDataBuffer = streamInManager.prepareForRead(1);
+
+ if (appDataBuffer.remaining() > 0 || streamDataBuffer.remaining() > 0) {
+ return true;
+ }
+
+ final ByteBuffer writableBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ final int bytesRead = channel.read(writableBuffer);
+ return (bytesRead > 0);
+ }
+
+ public int read() throws IOException {
+ final int bytesRead = read(oneByteBuffer);
+ if (bytesRead == -1) {
+ return -1;
+ }
+ return oneByteBuffer[0] & 0xFF;
+ }
+
+ public int read(final byte[] buffer) throws IOException {
+ return read(buffer, 0, buffer.length);
+ }
+
+ public int read(final byte[] buffer, final int offset, final int len) throws IOException {
+ logger.debug("{} Reading up to {} bytes of data", this, len);
+
+ if (!connected) {
+ connect();
+ }
+
+ int copied = copyFromAppDataBuffer(buffer, offset, len);
+ if (copied > 0) {
+ return copied;
+ }
+
+ appDataManager.clear();
+
+ while (true) {
+ // prepare buffers and call unwrap
+ final ByteBuffer streamInBuffer = streamInManager.prepareForRead(1);
+ SSLEngineResult unwrapResponse = null;
+ final ByteBuffer appDataBuffer = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ unwrapResponse = engine.unwrap(streamInBuffer, appDataBuffer);
+ logger.trace("{} When reading data, (handshake={}) Unwrap response: {}", new Object[]{this, handshaking, unwrapResponse});
+
+ switch (unwrapResponse.getStatus()) {
+ case BUFFER_OVERFLOW:
+ throw new SSLHandshakeException("Buffer Overflow, which is not allowed to happen from an unwrap");
+ case BUFFER_UNDERFLOW: {
+// appDataManager.prepareForRead(engine.getSession().getApplicationBufferSize());
+
+ final ByteBuffer writableInBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ final int bytesRead = readData(writableInBuffer);
+ if (bytesRead < 0) {
+ return -1;
+ }
+
+ continue;
+ }
+ case CLOSED:
+ throw new IOException("Channel is closed");
+ case OK: {
+ copied = copyFromAppDataBuffer(buffer, offset, len);
+ if (copied == 0) {
+ throw new IOException("Failed to decrypt data");
+ }
+ streamInManager.compact();
+ return copied;
+ }
+ }
+ }
+ }
+
+ public void write(final int data) throws IOException {
+ write(new byte[]{(byte) data}, 0, 1);
+ }
+
+ public void write(final byte[] data) throws IOException {
+ write(data, 0, data.length);
+ }
+
+ public void write(final byte[] data, final int offset, final int len) throws IOException {
+ logger.debug("{} Writing {} bytes of data", this, len);
+
+ if (!connected) {
+ connect();
+ }
+
+ int iterations = len / MAX_WRITE_SIZE;
+ if (len % MAX_WRITE_SIZE > 0) {
+ iterations++;
+ }
+
+ for (int i = 0; i < iterations; i++) {
+ streamOutManager.clear();
+ final int itrOffset = offset + i * MAX_WRITE_SIZE;
+ final int itrLen = Math.min(len - itrOffset, MAX_WRITE_SIZE);
+ final ByteBuffer byteBuffer = ByteBuffer.wrap(data, itrOffset, itrLen);
+
+ final BufferStateManager buffMan = new BufferStateManager(byteBuffer, Direction.READ);
+ final Status status = encryptAndWriteFully(buffMan);
+ switch (status) {
+ case BUFFER_OVERFLOW:
+ streamOutManager.ensureSize(engine.getSession().getPacketBufferSize());
+ appDataManager.ensureSize(engine.getSession().getApplicationBufferSize());
+ continue;
+ case OK:
+ continue;
+ case CLOSED:
+ throw new IOException("Channel is closed");
+ case BUFFER_UNDERFLOW:
+ throw new AssertionError("Got Buffer Underflow but should not have...");
+ }
+ }
+ }
+
+ public void interrupt() {
+ this.interrupted = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
new file mode 100644
index 0000000..154bd08
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class SSLSocketChannelInputStream extends InputStream {
+
+ private final SSLSocketChannel channel;
+
+ public SSLSocketChannelInputStream(final SSLSocketChannel channel) {
+ this.channel = channel;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return channel.read();
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return channel.read(b);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ return channel.read(b, off, len);
+ }
+
+ /**
+ * Closes the underlying SSLSocketChannel, which will also close the
+ * OutputStream and connection
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return channel.available();
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ return available() > 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
new file mode 100644
index 0000000..ce4e420
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class SSLSocketChannelOutputStream extends OutputStream {
+
+ private final SSLSocketChannel channel;
+
+ public SSLSocketChannelOutputStream(final SSLSocketChannel channel) {
+ this.channel = channel;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ channel.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ channel.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ channel.write(b, off, len);
+ }
+
+ /**
+ * Closes the underlying SSLSocketChannel, which also will close the
+ * InputStream and the connection
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java
new file mode 100644
index 0000000..aaf37ea
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.InputStream;
+
+/**
+ * This class is a slight modification of the BufferedInputStream in the java.io
+ * package. The modification is that this implementation does not provide
+ * synchronization on method calls, which means that this class is not suitable
+ * for use by multiple threads. However, the absence of these synchronized
+ * blocks results in potentially much better performance.
+ */
+public class BufferedInputStream extends java.io.BufferedInputStream {
+
+ public BufferedInputStream(final InputStream in) {
+ super(in);
+ }
+
+ public BufferedInputStream(final InputStream in, final int size) {
+ super(in, size);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java
new file mode 100644
index 0000000..eadfcab
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * This class is a slight modification of the
+ * {@link java.io.BufferedOutputStream} class. This implementation differs in
+ * that it does not mark methods as synchronized. This means that this class is
+ * not suitable for writing by multiple concurrent threads. However, the removal
+ * of the synchronized keyword results in potentially much better performance.
+ */
+public class BufferedOutputStream extends FilterOutputStream {
+
+ /**
+ * The internal buffer where data is stored.
+ */
+ protected byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer. This value is always in the
+ * range <tt>0</tt> through <tt>buf.length</tt>; elements
+ * <tt>buf[0]</tt> through <tt>buf[count-1]</tt> contain valid byte data.
+ */
+ protected int count;
+
+ /**
+ * Creates a new buffered output stream to write data to the specified
+ * underlying output stream.
+ *
+ * @param out the underlying output stream.
+ */
+ public BufferedOutputStream(OutputStream out) {
+ this(out, 8192);
+ }
+
+ /**
+ * Creates a new buffered output stream to write data to the specified
+ * underlying output stream with the specified buffer size.
+ *
+ * @param out the underlying output stream.
+ * @param size the buffer size.
+ * @exception IllegalArgumentException if size <= 0.
+ */
+ public BufferedOutputStream(OutputStream out, int size) {
+ super(out);
+ if (size <= 0) {
+ throw new IllegalArgumentException("Buffer size <= 0");
+ }
+ buf = new byte[size];
+ }
+
+ /**
+ * Flush the internal buffer
+ */
+ private void flushBuffer() throws IOException {
+ if (count > 0) {
+ out.write(buf, 0, count);
+ count = 0;
+ }
+ }
+
+ /**
+ * Writes the specified byte to this buffered output stream.
+ *
+ * @param b the byte to be written.
+ * @exception IOException if an I/O error occurs.
+ */
+ @Override
+ public void write(int b) throws IOException {
+ if (count >= buf.length) {
+ flushBuffer();
+ }
+ buf[count++] = (byte) b;
+ }
+
+ /**
+ * Writes <code>len</code> bytes from the specified byte array starting at
+ * offset <code>off</code> to this buffered output stream.
+ *
+ * <p>
+ * Ordinarily this method stores bytes from the given array into this
+ * stream's buffer, flushing the buffer to the underlying output stream as
+ * needed. If the requested length is at least as large as this stream's
+ * buffer, however, then this method will flush the buffer and write the
+ * bytes directly to the underlying output stream. Thus redundant
+ * <code>BufferedOutputStream</code>s will not copy data unnecessarily.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @exception IOException if an I/O error occurs.
+ */
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if (len >= buf.length) {
+ /* If the request length exceeds the size of the output buffer,
+ flush the output buffer and then write the data directly.
+ In this way buffered streams will cascade harmlessly. */
+ flushBuffer();
+ out.write(b, off, len);
+ return;
+ }
+ if (len >= buf.length - count) {
+ flushBuffer();
+ }
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ /**
+ * Flushes this buffered output stream. This forces any buffered output
+ * bytes to be written out to the underlying output stream.
+ *
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public void flush() throws IOException {
+ flushBuffer();
+ out.flush();
+ }
+}
[21/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
new file mode 100644
index 0000000..41a0557
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
@@ -0,0 +1,623 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file;
+
+import java.io.BufferedInputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.slf4j.Logger;
+
+/**
+ * A utility class containing a few useful static methods to do typical IO
+ * operations.
+ *
+ * @author unattributed
+ */
+public class FileUtils {
+
+ public static final long TRANSFER_CHUNK_SIZE_BYTES = 1024 * 1024 * 8; //8 MB chunks
+ public static final long MILLIS_BETWEEN_ATTEMPTS = 50L;
+
+ /**
+ * Closes the given closeable quietly - no logging, no exceptions...
+ *
+ * @param closeable
+ */
+ public static void closeQuietly(final Closeable closeable) {
+ if (null != closeable) {
+ try {
+ closeable.close();
+ } catch (final IOException io) {/*IGNORE*/
+
+ }
+ }
+ }
+
+ /**
+ * Releases the given lock quietly - no logging, no exception
+ *
+ * @param lock
+ */
+ public static void releaseQuietly(final FileLock lock) {
+ if (null != lock) {
+ try {
+ lock.release();
+ } catch (final IOException io) {
+ /*IGNORE*/
+ }
+ }
+ }
+
+ public static void ensureDirectoryExistAndCanAccess(final File dir) throws IOException {
+ if (dir.exists() && !dir.isDirectory()) {
+ throw new IOException(dir.getAbsolutePath() + " is not a directory");
+ } else if (!dir.exists()) {
+ final boolean made = dir.mkdirs();
+ if (!made) {
+ throw new IOException(dir.getAbsolutePath() + " could not be created");
+ }
+ }
+ if (!(dir.canRead() && dir.canWrite())) {
+ throw new IOException(dir.getAbsolutePath() + " directory does not have read/write privilege");
+ }
+ }
+
+ /**
+ * Deletes the given file. If the given file exists but could not be deleted
+ * this will be printed as a warning to the given logger
+ *
+ * @param file
+ * @param logger
+ * @return
+ */
+ public static boolean deleteFile(final File file, final Logger logger) {
+ return FileUtils.deleteFile(file, logger, 1);
+ }
+
+ /**
+ * Deletes the given file. If the given file exists but could not be deleted
+ * this will be printed as a warning to the given logger
+ *
+ * @param file
+ * @param logger
+ * @param attempts indicates how many times an attempt to delete should be
+ * made
+ * @return true if given file no longer exists
+ */
+ public static boolean deleteFile(final File file, final Logger logger, final int attempts) {
+ if (file == null) {
+ return false;
+ }
+ boolean isGone = false;
+ try {
+ if (file.exists()) {
+ final int effectiveAttempts = Math.max(1, attempts);
+ for (int i = 0; i < effectiveAttempts && !isGone; i++) {
+ isGone = file.delete() || !file.exists();
+ if (!isGone && (effectiveAttempts - i) > 1) {
+ FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS);
+ }
+ }
+ if (!isGone && logger != null) {
+ logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath());
+ }
+ }
+ } catch (final Throwable t) {
+ if (logger != null) {
+ logger.warn("Unable to delete file: '" + file.getAbsolutePath() + "' due to " + t);
+ }
+ }
+ return isGone;
+ }
+
+ /**
+ * Deletes all of the given files. If any exist and cannot be deleted that
+ * will be printed at warn to the given logger.
+ *
+ * @param files can be null
+ * @param logger can be null
+ */
+ public static void deleteFile(final List<File> files, final Logger logger) {
+ FileUtils.deleteFile(files, logger, 1);
+ }
+
+ /**
+ * Deletes all of the given files. If any exist and cannot be deleted that
+ * will be printed at warn to the given logger.
+ *
+ * @param files can be null
+ * @param logger can be null
+ * @param attempts indicates how many times an attempt should be made to
+ * delete each file
+ */
+ public static void deleteFile(final List<File> files, final Logger logger, final int attempts) {
+ if (null == files || files.isEmpty()) {
+ return;
+ }
+ final int effectiveAttempts = Math.max(1, attempts);
+ for (final File file : files) {
+ try {
+ boolean isGone = false;
+ for (int i = 0; i < effectiveAttempts && !isGone; i++) {
+ isGone = file.delete() || !file.exists();
+ if (!isGone && (effectiveAttempts - i) > 1) {
+ FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS);
+ }
+ }
+ if (!isGone && logger != null) {
+ logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath());
+ }
+ } catch (final Throwable t) {
+ if (null != logger) {
+ logger.warn("Unable to delete file given from path: '" + file.getPath() + "' due to " + t);
+ }
+ }
+ }
+ }
+
+ /**
+ * Deletes all files (not directories..) in the given directory (non
+ * recursive) that match the given filename filter. If any file cannot be
+ * deleted then this is printed at warn to the given logger.
+ *
+ * @param directory
+ * @param filter if null then no filter is used
+ * @param logger
+ */
+ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger) {
+ FileUtils.deleteFilesInDir(directory, filter, logger, false);
+ }
+
+ /**
+ * Deletes all files (not directories) in the given directory (recursive)
+ * that match the given filename filter. If any file cannot be deleted then
+ * this is printed at warn to the given logger.
+ *
+ * @param directory
+ * @param filter if null then no filter is used
+ * @param logger
+ * @param recurse
+ */
+ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse) {
+ FileUtils.deleteFilesInDir(directory, filter, logger, recurse, false);
+ }
+
+ /**
+ * Deletes all files (not directories) in the given directory (recursive)
+ * that match the given filename filter. If any file cannot be deleted then
+ * this is printed at warn to the given logger.
+ *
+ * @param directory
+ * @param filter if null then no filter is used
+ * @param logger
+ * @param recurse
+ * @param deleteEmptyDirectories default is false; if true will delete
+ * directories found that are empty
+ */
+ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse, final boolean deleteEmptyDirectories) {
+ // ensure the specified directory is actually a directory and that it exists
+ if (null != directory && directory.isDirectory()) {
+ final File ingestFiles[] = directory.listFiles();
+ for (File ingestFile : ingestFiles) {
+ boolean process = (filter == null) ? true : filter.accept(directory, ingestFile.getName());
+ if (ingestFile.isFile() && process) {
+ FileUtils.deleteFile(ingestFile, logger, 3);
+ }
+ if (ingestFile.isDirectory() && recurse) {
+ FileUtils.deleteFilesInDir(ingestFile, filter, logger, recurse, deleteEmptyDirectories);
+ if (deleteEmptyDirectories && ingestFile.list().length == 0) {
+ FileUtils.deleteFile(ingestFile, logger, 3);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Deletes given files.
+ *
+ * @param files
+ * @param recurse will recurse
+ * @throws IOException
+ */
+ public static void deleteFiles(final Collection<File> files, final boolean recurse) throws IOException {
+ for (final File file : files) {
+ FileUtils.deleteFile(file, recurse);
+ }
+ }
+
+ public static void deleteFile(final File file, final boolean recurse) throws IOException {
+ if (file.isDirectory() && recurse) {
+ FileUtils.deleteFiles(Arrays.asList(file.listFiles()), recurse);
+ }
+ //now delete the file itself regardless of whether it is plain file or a directory
+ if (!FileUtils.deleteFile(file, null, 5)) {
+ throw new IOException("Unable to delete " + file.getAbsolutePath());
+ }
+ }
+
+ /**
+ * Randomly generates a sequence of bytes and overwrites the contents of the
+ * file a number of times. The file is then deleted.
+ *
+ * @param file File to be overwritten a number of times and, ultimately,
+ * deleted
+ * @param passes Number of times file should be overwritten
+ * @throws IOException if something makes shredding or deleting a problem
+ */
+ public static void shredFile(final File file, final int passes)
+ throws IOException {
+ final Random generator = new Random();
+ final long fileLength = file.length();
+ final int byteArraySize = (int) Math.min(fileLength, 1048576); // 1MB
+ final byte[] b = new byte[byteArraySize];
+ final long numOfRandomWrites = (fileLength / b.length) + 1;
+ final FileOutputStream fos = new FileOutputStream(file);
+ try {
+ // Over write file contents (passes) times
+ final FileChannel channel = fos.getChannel();
+ for (int i = 0; i < passes; i++) {
+ generator.nextBytes(b);
+ for (int j = 0; j <= numOfRandomWrites; j++) {
+ fos.write(b);
+ }
+ fos.flush();
+ channel.position(0);
+ }
+ // Write out "0" for each byte in the file
+ Arrays.fill(b, (byte) 0);
+ for (int j = 0; j < numOfRandomWrites; j++) {
+ fos.write(b);
+ }
+ fos.flush();
+ fos.close();
+ // Try to delete the file a few times
+ if (!FileUtils.deleteFile(file, null, 5)) {
+ throw new IOException("Failed to delete file after shredding");
+ }
+
+ } finally {
+ FileUtils.closeQuietly(fos);
+ }
+ }
+
+ public static long copy(final InputStream in, final OutputStream out) throws IOException {
+ final byte[] buffer = new byte[65536];
+ long copied = 0L;
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ copied += len;
+ }
+
+ return copied;
+ }
+
+ public static long copyBytes(final byte[] bytes, final File destination, final boolean lockOutputFile) throws FileNotFoundException, IOException {
+ FileOutputStream fos = null;
+ FileLock outLock = null;
+ long fileSize = 0L;
+ try {
+ fos = new FileOutputStream(destination);
+ final FileChannel out = fos.getChannel();
+ if (lockOutputFile) {
+ outLock = out.tryLock(0, Long.MAX_VALUE, false);
+ if (null == outLock) {
+ throw new IOException("Unable to obtain exclusive file lock for: " + destination.getAbsolutePath());
+ }
+ }
+ fos.write(bytes);
+ fos.flush();
+ fileSize = bytes.length;
+ } finally {
+ FileUtils.releaseQuietly(outLock);
+ FileUtils.closeQuietly(fos);
+ }
+ return fileSize;
+ }
+
+ /**
+ * Copies the given source file to the given destination file. The given
+ * destination will be overwritten if it already exists.
+ *
+ * @param source
+ * @param destination
+ * @param lockInputFile if true will lock input file during copy; if false
+ * will not
+ * @param lockOutputFile if true will lock output file during copy; if false
+ * will not
+ * @param move if true will perform what is effectively a move operation
+ * rather than a pure copy. This allows for potentially highly efficient
+ * movement of the file but if not possible this will revert to a copy then
+ * delete behavior. If false, then the file is copied and the source file is
+ * retained. If a true rename/move occurs then no lock is held during that
+ * time.
+ * @param logger if failures occur, they will be logged to this logger if
+ * possible. If this logger is null, an IOException will instead be thrown,
+ * indicating the problem.
+ * @return long number of bytes copied
+ * @throws FileNotFoundException if the source file could not be found
+ * @throws IOException
+ * @throws SecurityException if a security manager denies the needed file
+ * operations
+ */
+ public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final boolean move, final Logger logger) throws FileNotFoundException, IOException {
+
+ FileInputStream fis = null;
+ FileOutputStream fos = null;
+ FileLock inLock = null;
+ FileLock outLock = null;
+ long fileSize = 0L;
+ if (!source.canRead()) {
+ throw new IOException("Must at least have read permission");
+
+ }
+ if (move && source.renameTo(destination)) {
+ fileSize = destination.length();
+ } else {
+ try {
+ fis = new FileInputStream(source);
+ fos = new FileOutputStream(destination);
+ final FileChannel in = fis.getChannel();
+ final FileChannel out = fos.getChannel();
+ if (lockInputFile) {
+ inLock = in.tryLock(0, Long.MAX_VALUE, true);
+ if (null == inLock) {
+ throw new IOException("Unable to obtain shared file lock for: " + source.getAbsolutePath());
+ }
+ }
+ if (lockOutputFile) {
+ outLock = out.tryLock(0, Long.MAX_VALUE, false);
+ if (null == outLock) {
+ throw new IOException("Unable to obtain exclusive file lock for: " + destination.getAbsolutePath());
+ }
+ }
+ long bytesWritten = 0;
+ do {
+ bytesWritten += out.transferFrom(in, bytesWritten, TRANSFER_CHUNK_SIZE_BYTES);
+ fileSize = in.size();
+ } while (bytesWritten < fileSize);
+ out.force(false);
+ FileUtils.closeQuietly(fos);
+ FileUtils.closeQuietly(fis);
+ fos = null;
+ fis = null;
+ if (move && !FileUtils.deleteFile(source, null, 5)) {
+ if (logger == null) {
+ FileUtils.deleteFile(destination, null, 5);
+ throw new IOException("Could not remove file " + source.getAbsolutePath());
+ } else {
+ logger.warn("Configured to delete source file when renaming/move not successful. However, unable to delete file at: " + source.getAbsolutePath());
+ }
+ }
+ } finally {
+ FileUtils.releaseQuietly(inLock);
+ FileUtils.releaseQuietly(outLock);
+ FileUtils.closeQuietly(fos);
+ FileUtils.closeQuietly(fis);
+ }
+ }
+ return fileSize;
+ }
+
+ /**
+ * Copies the given source file to the given destination file. The given
+ * destination will be overwritten if it already exists.
+ *
+ * @param source
+ * @param destination
+ * @param lockInputFile if true will lock input file during copy; if false
+ * will not
+ * @param lockOutputFile if true will lock output file during copy; if false
+ * will not
+ * @param logger
+ * @return long number of bytes copied
+ * @throws FileNotFoundException if the source file could not be found
+ * @throws IOException
+ * @throws SecurityException if a security manager denies the needed file
+ * operations
+ */
+ public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final Logger logger) throws FileNotFoundException, IOException {
+ return FileUtils.copyFile(source, destination, lockInputFile, lockOutputFile, false, logger);
+ }
+
+ public static long copyFile(final File source, final OutputStream stream, final boolean closeOutputStream, final boolean lockInputFile) throws FileNotFoundException, IOException {
+ FileInputStream fis = null;
+ FileLock inLock = null;
+ long fileSize = 0L;
+ try {
+ fis = new FileInputStream(source);
+ final FileChannel in = fis.getChannel();
+ if (lockInputFile) {
+ inLock = in.tryLock(0, Long.MAX_VALUE, true);
+ if (inLock == null) {
+ throw new IOException("Unable to obtain exclusive file lock for: " + source.getAbsolutePath());
+ }
+
+ }
+
+ byte[] buffer = new byte[1 << 18]; //256 KB
+ int bytesRead = -1;
+ while ((bytesRead = fis.read(buffer)) != -1) {
+ stream.write(buffer, 0, bytesRead);
+ }
+ in.force(false);
+ stream.flush();
+ fileSize = in.size();
+ } finally {
+ FileUtils.releaseQuietly(inLock);
+ FileUtils.closeQuietly(fis);
+ if (closeOutputStream) {
+ FileUtils.closeQuietly(stream);
+ }
+ }
+ return fileSize;
+ }
+
+ public static long copyFile(final InputStream stream, final File destination, final boolean closeInputStream, final boolean lockOutputFile) throws FileNotFoundException, IOException {
+ final Path destPath = destination.toPath();
+ final long size = Files.copy(stream, destPath);
+ if (closeInputStream) {
+ stream.close();
+ }
+ return size;
+ }
+
+ /**
+ * Renames the given file from the source path to the destination path. This
+ * handles multiple attempts. This should only be used to rename within a
+ * given directory. Renaming across directories might not work well. See the
+ * <code>File.renameTo</code> for more information.
+ *
+ * @param source the file to rename
+ * @param destination the file path to rename to
+ * @param maxAttempts the max number of attempts to attempt the rename
+ * @throws IOException if rename isn't successful
+ */
+ public static void renameFile(final File source, final File destination, final int maxAttempts) throws IOException {
+ FileUtils.renameFile(source, destination, maxAttempts, false);
+ }
+
+ /**
+ * Renames the given file from the source path to the destination path. This
+ * handles multiple attempts. This should only be used to rename within a
+ * given directory. Renaming across directories might not work well. See the
+ * <code>File.renameTo</code> for more information.
+ *
+ * @param source the file to rename
+ * @param destination the file path to rename to
+ * @param maxAttempts the max number of attempts to attempt the rename
+ * @param replace if true and a rename attempt fails will check if a file is
+ * already at the destination path. If so it will delete that file and
+ * attempt the rename according the remaining maxAttempts. If false, any
+ * conflicting files will be left as they were and the rename attempts will
+ * fail if conflicting.
+ * @throws IOException if rename isn't successful
+ */
+ public static void renameFile(final File source, final File destination, final int maxAttempts, final boolean replace) throws IOException {
+ final int attempts = (replace || maxAttempts < 1) ? Math.max(2, maxAttempts) : maxAttempts;
+ boolean renamed = false;
+ for (int i = 0; i < attempts; i++) {
+ renamed = source.renameTo(destination);
+ if (!renamed) {
+ FileUtils.deleteFile(destination, null, 5);
+ } else {
+ break; //rename has succeeded
+ }
+ }
+ if (!renamed) {
+ throw new IOException("Attempted " + maxAttempts + " times but unable to rename from \'" + source.getPath() + "\' to \'" + destination.getPath() + "\'");
+
+ }
+ }
+
+ public static void sleepQuietly(final long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (final InterruptedException ex) {
+ /* do nothing */
+ }
+ }
+
+ /**
+ * Syncs a primary copy of a file with the copy in the restore directory. If
+ * the restore directory does not have a file and the primary has a file,
+ * the the primary's file is copied to the restore directory. Else if the
+ * restore directory has a file, but the primary does not, then the
+ * restore's file is copied to the primary directory. Else if the primary
+ * file is different than the restore file, then an IllegalStateException is
+ * thrown. Otherwise, if neither file exists, then no syncing is performed.
+ *
+ * @param primaryFile the primary file
+ * @param restoreFile the restore file
+ * @param logger a logger
+ * @throws IOException if an I/O problem was encountered during syncing
+ * @throws IllegalStateException if the primary and restore copies exist but
+ * are different
+ */
+ public static void syncWithRestore(final File primaryFile, final File restoreFile, final Logger logger)
+ throws IOException {
+
+ if (primaryFile.exists() && !restoreFile.exists()) {
+ // copy primary file to restore
+ copyFile(primaryFile, restoreFile, false, false, logger);
+ } else if (restoreFile.exists() && !primaryFile.exists()) {
+ // copy restore file to primary
+ copyFile(restoreFile, primaryFile, false, false, logger);
+ } else if (primaryFile.exists() && restoreFile.exists() && !isSame(primaryFile, restoreFile)) {
+ throw new IllegalStateException(String.format("Primary file '%s' is different than restore file '%s'",
+ primaryFile.getAbsoluteFile(), restoreFile.getAbsolutePath()));
+ }
+ }
+
+ /**
+ * Returns true if the given files are the same according to their MD5 hash.
+ *
+ * @param file1 a file
+ * @param file2 a file
+ * @return true if the files are the same; false otherwise
+ * @throws IOException if the MD5 hash could not be computed
+ */
+ public static boolean isSame(final File file1, final File file2) throws IOException {
+ return Arrays.equals(computeMd5Digest(file1), computeMd5Digest(file2));
+ }
+
+ /**
+ * Returns the MD5 hash of the given file.
+ *
+ * @param file a file
+ * @return the MD5 hash
+ * @throws IOException if the MD5 hash could not be computed
+ */
+ public static byte[] computeMd5Digest(final File file) throws IOException {
+ final MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance("MD5");
+ } catch (final NoSuchAlgorithmException nsae) {
+ throw new IOException(nsae);
+ }
+
+ try (final FileInputStream fis = new FileInputStream(file)) {
+ int len;
+ final byte[] buffer = new byte[8192];
+ while ((len = fis.read(buffer)) > -1) {
+ if (len > 0) {
+ digest.update(buffer, 0, len);
+ }
+ }
+ }
+ return digest.digest();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java
new file mode 100644
index 0000000..6f9c616
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An {@link UpdateMonitor} that combines multiple <code>UpdateMonitor</code>s
+ * such that it will indicate a change in a file only if ALL sub-monitors
+ * indicate a change. The sub-monitors will be applied in the order given and if
+ * any indicates that the state has not changed, the subsequent sub-monitors may
+ * not be given a chance to run
+ */
+public class CompoundUpdateMonitor implements UpdateMonitor {
+
+ private final List<UpdateMonitor> monitors;
+
+ public CompoundUpdateMonitor(final UpdateMonitor first, final UpdateMonitor... others) {
+ monitors = new ArrayList<>();
+ monitors.add(first);
+ for (final UpdateMonitor monitor : others) {
+ monitors.add(monitor);
+ }
+ }
+
+ @Override
+ public Object getCurrentState(final Path path) throws IOException {
+ return new DeferredMonitorAction(monitors, path);
+ }
+
+ private static class DeferredMonitorAction {
+
+ private static final Object NON_COMPUTED_VALUE = new Object();
+
+ private final List<UpdateMonitor> monitors;
+ private final Path path;
+
+ private final Object[] preCalculated;
+
+ public DeferredMonitorAction(final List<UpdateMonitor> monitors, final Path path) {
+ this.monitors = monitors;
+ this.path = path;
+ preCalculated = new Object[monitors.size()];
+
+ for (int i = 0; i < preCalculated.length; i++) {
+ preCalculated[i] = NON_COMPUTED_VALUE;
+ }
+ }
+
+ private Object getCalculatedValue(final int i) throws IOException {
+ if (preCalculated[i] == NON_COMPUTED_VALUE) {
+ preCalculated[i] = monitors.get(i).getCurrentState(path);
+ }
+
+ return preCalculated[i];
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ // must return true unless ALL DeferredMonitorAction's indicate that they are different
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof DeferredMonitorAction)) {
+ return false;
+ }
+
+ final DeferredMonitorAction other = (DeferredMonitorAction) obj;
+ try {
+ // Go through each UpdateMonitor's value and check if the value has changed.
+ for (int i = 0; i < preCalculated.length; i++) {
+ final Object mine = getCalculatedValue(i);
+ final Object theirs = other.getCalculatedValue(i);
+
+ if (mine == theirs) {
+ // same
+ return true;
+ }
+
+ if (mine == null && theirs == null) {
+ // same
+ return true;
+ }
+
+ if (mine.equals(theirs)) {
+ return true;
+ }
+ }
+ } catch (final IOException e) {
+ return false;
+ }
+
+ // No DeferredMonitorAction was the same as last time. Therefore, it's not equal
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java
new file mode 100644
index 0000000..e6be558
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class LastModifiedMonitor implements UpdateMonitor {
+
+ @Override
+ public Object getCurrentState(final Path path) throws IOException {
+ return Files.getLastModifiedTime(path);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java
new file mode 100644
index 0000000..8dea4bf
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class MD5SumMonitor implements UpdateMonitor {
+
+ @Override
+ public Object getCurrentState(final Path path) throws IOException {
+ final MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance("MD5");
+ } catch (final NoSuchAlgorithmException nsae) {
+ throw new AssertionError(nsae);
+ }
+
+ try (final FileInputStream fis = new FileInputStream(path.toFile())) {
+ int len;
+ final byte[] buffer = new byte[8192];
+ while ((len = fis.read(buffer)) > -1) {
+ if (len > 0) {
+ digest.update(buffer, 0, len);
+ }
+ }
+ }
+
+ // Return a ByteBuffer instead of byte[] because we want equals() to do a deep equality
+ return ByteBuffer.wrap(digest.digest());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java
new file mode 100644
index 0000000..e0089c1
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Allows the user to configure a {@link java.nio.file.Path Path} to watch for
+ * modifications and periodically poll to check if the file has been modified
+ */
+public class SynchronousFileWatcher {
+
+ private final Path path;
+ private final long checkUpdateMillis;
+ private final UpdateMonitor monitor;
+ private final AtomicReference<StateWrapper> lastState;
+ private final Lock resourceLock = new ReentrantLock();
+
+ public SynchronousFileWatcher(final Path path, final UpdateMonitor monitor) {
+ this(path, monitor, 0L);
+ }
+
+ public SynchronousFileWatcher(final Path path, final UpdateMonitor monitor, final long checkMillis) {
+ if (checkMillis < 0) {
+ throw new IllegalArgumentException();
+ }
+
+ this.path = path;
+ checkUpdateMillis = checkMillis;
+ this.monitor = monitor;
+
+ Object currentState;
+ try {
+ currentState = monitor.getCurrentState(path);
+ } catch (final IOException e) {
+ currentState = null;
+ }
+
+ this.lastState = new AtomicReference<>(new StateWrapper(currentState));
+ }
+
+ /**
+ * Checks if the file has been updated according to the configured
+ * {@link UpdateMonitor} and resets the state
+ *
+ * @return
+ * @throws IOException
+ */
+ public boolean checkAndReset() throws IOException {
+ if (checkUpdateMillis <= 0) { // if checkUpdateMillis <= 0, always check
+ return checkForUpdate();
+ } else {
+ final StateWrapper stateWrapper = lastState.get();
+ if (stateWrapper.getTimestamp() < System.currentTimeMillis() - checkUpdateMillis) {
+ return checkForUpdate();
+ }
+ return false;
+ }
+ }
+
+ private boolean checkForUpdate() throws IOException {
+ if (resourceLock.tryLock()) {
+ try {
+ final StateWrapper wrapper = lastState.get();
+ final Object newState = monitor.getCurrentState(path);
+ if (newState == null && wrapper.getState() == null) {
+ return false;
+ }
+ if (newState == null || wrapper.getState() == null) {
+ lastState.set(new StateWrapper(newState));
+ return true;
+ }
+
+ final boolean unmodified = newState.equals(wrapper.getState());
+ if (!unmodified) {
+ lastState.set(new StateWrapper(newState));
+ }
+ return !unmodified;
+ } finally {
+ resourceLock.unlock();
+ }
+ } else {
+ return false;
+ }
+ }
+
+ private static class StateWrapper {
+
+ private final Object state;
+ private final long timestamp;
+
+ public StateWrapper(final Object state) {
+ this.state = state;
+ this.timestamp = System.currentTimeMillis();
+ }
+
+ public Object getState() {
+ return state;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java
new file mode 100644
index 0000000..20ed1dd
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public interface UpdateMonitor {
+
+ Object getCurrentState(Path path) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java
new file mode 100644
index 0000000..59b444a
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+
+import org.apache.nifi.util.search.ahocorasick.SearchState;
+
+/**
+ * Defines an interface to search for content given a set of search terms. Any
+ * implementation of search must be thread safe.
+ *
+ * @author
+ * @param <T>
+ */
+public interface Search<T> {
+
+ /**
+ * Establishes the dictionary of terms which will be searched in subsequent
+ * search calls. This can be called only once
+ *
+ * @param terms
+ */
+ void initializeDictionary(Set<SearchTerm<T>> terms);
+
+ /**
+ * Searches the given input stream for matches between the already specified
+ * dictionary and the contents scanned.
+ *
+ * @param haystack
+ * @param findAll if true will find all matches if false will find only the
+ * first match
+ * @return SearchState containing results Map might be empty which indicates
+ * no matches found but will not be null
+ * @throws IOException Thrown for any exceptions occurring while searching.
+ * @throws IllegalStateException if the dictionary has not yet been
+ * initialized
+ */
+ SearchState<T> search(InputStream haystack, boolean findAll) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
new file mode 100644
index 0000000..62de964
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search;
+
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+/**
+ * This is an immutable thread safe object representing a search term
+ *
+ * @author
+ * @param <T>
+ */
+public class SearchTerm<T> {
+
+ private final byte[] bytes;
+ private final int hashCode;
+ private final T reference;
+
+ /**
+ * Constructs a SearchTerm. Defensively copies the given byte array
+ *
+ * @param bytes
+ * @throws IllegalArgument exception if given bytes are null or 0 length
+ */
+ public SearchTerm(final byte[] bytes) {
+ this(bytes, true, null);
+ }
+
+ /**
+ * Constructs a search term. Optionally performs a defensive copy of the
+ * given byte array. If the caller indicates a defensive copy is not
+ * necessary then they must not change the given arrays state any longer
+ *
+ * @param bytes
+ * @param defensiveCopy
+ * @param reference
+ */
+ public SearchTerm(final byte[] bytes, final boolean defensiveCopy, final T reference) {
+ if (bytes == null || bytes.length == 0) {
+ throw new IllegalArgumentException();
+ }
+ if (defensiveCopy) {
+ this.bytes = Arrays.copyOf(bytes, bytes.length);
+ } else {
+ this.bytes = bytes;
+ }
+ this.hashCode = Arrays.hashCode(this.bytes);
+ this.reference = reference;
+ }
+
+ public int get(final int index) {
+ return bytes[index] & 0xff;
+ }
+
+ /**
+ * @return size in of search term in bytes
+ */
+ public int size() {
+ return bytes.length;
+ }
+
+ /**
+ * @return reference object for this given search term
+ */
+ public T getReference() {
+ return reference;
+ }
+
+ /**
+ * Determines if the given window starts with the same bytes as this term
+ *
+ * @param window Current window of bytes from the haystack being evaluated.
+ * @param windowLength The length of the window to consider
+ * @return true if this term starts with the same bytes of the given window
+ */
+ public boolean startsWith(byte[] window, int windowLength) {
+ if (windowLength > window.length) {
+ throw new IndexOutOfBoundsException();
+ }
+ if (bytes.length < windowLength) {
+ return false;
+ }
+ for (int i = 0; i < bytes.length && i < windowLength; i++) {
+ if (bytes[i] != window[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @return a defensive copy of the internal byte structure
+ */
+ public byte[] getBytes() {
+ return Arrays.copyOf(bytes, bytes.length);
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final SearchTerm other = (SearchTerm) obj;
+ if (this.hashCode != other.hashCode) {
+ return false;
+ }
+ return Arrays.equals(this.bytes, other.bytes);
+ }
+
+ @Override
+ public String toString() {
+ return new String(bytes);
+ }
+
+ public String toString(final Charset charset) {
+ return new String(bytes, charset);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
new file mode 100644
index 0000000..3b8afaf
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search.ahocorasick;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.Set;
+
+import org.apache.nifi.util.search.Search;
+import org.apache.nifi.util.search.SearchTerm;
+
+public class AhoCorasick<T> implements Search<T> {
+
+ private Node root = null;
+
+ /**
+ * Constructs a new search object.
+ *
+ * @throws IllegalArgumentException if given terms are null or empty
+ */
+ public AhoCorasick() {
+ }
+
+ @Override
+ public void initializeDictionary(final Set<SearchTerm<T>> terms) {
+ if (root != null) {
+ throw new IllegalStateException();
+ }
+ root = new Node();
+ if (terms == null || terms.isEmpty()) {
+ throw new IllegalArgumentException();
+ }
+ for (final SearchTerm<T> term : terms) {
+ int i = 0;
+ Node nextNode = root;
+ while (true) {
+ nextNode = addMatch(term, i, nextNode);
+ if (nextNode == null) {
+ break; //we're done
+ }
+ i++;
+ }
+ }
+ initialize();
+ }
+
+ private Node addMatch(final SearchTerm<T> term, final int offset, final Node current) {
+ final int index = term.get(offset);
+ boolean atEnd = (offset == (term.size() - 1));
+ if (current.getNeighbor(index) == null) {
+ if (atEnd) {
+ current.setNeighbor(new Node(term), index);
+ return null;
+ }
+ current.setNeighbor(new Node(), index);
+ } else if (atEnd) {
+ current.getNeighbor(index).setMatchingTerm(term);
+ return null;
+ }
+ return current.getNeighbor(index);
+ }
+
+ private void initialize() {
+ //perform bgs to build failure links
+ final Queue<Node> queue = new LinkedList<>();
+ queue.add(root);
+ root.setFailureNode(null);
+ while (!queue.isEmpty()) {
+ final Node current = queue.poll();
+ for (int i = 0; i < 256; i++) {
+ final Node next = current.getNeighbor(i);
+ if (next != null) {
+ //traverse failure to get state
+ Node fail = current.getFailureNode();
+ while ((fail != null) && fail.getNeighbor(i) == null) {
+ fail = fail.getFailureNode();
+ }
+ if (fail != null) {
+ next.setFailureNode(fail.getNeighbor(i));
+ } else {
+ next.setFailureNode(root);
+ }
+ queue.add(next);
+ }
+ }
+ }
+ }
+
+ @Override
+ public SearchState search(final InputStream stream, final boolean findAll) throws IOException {
+ return search(stream, findAll, null);
+ }
+
+ private SearchState search(final InputStream stream, final boolean findAll, final SearchState state) throws IOException {
+ if (root == null) {
+ throw new IllegalStateException();
+ }
+ final SearchState<T> currentState = (state == null) ? new SearchState(root) : state;
+ if (!findAll && currentState.foundMatch()) {
+ throw new IllegalStateException("A match has already been found yet we're being asked to keep searching");
+ }
+ Node current = currentState.getCurrentNode();
+ int currentChar;
+ while ((currentChar = stream.read()) >= 0) {
+ currentState.incrementBytesRead(1L);
+ Node next = current.getNeighbor(currentChar);
+ if (next == null) {
+ next = current.getFailureNode();
+ while ((next != null) && next.getNeighbor(currentChar) == null) {
+ next = next.getFailureNode();
+ }
+ if (next != null) {
+ next = next.getNeighbor(currentChar);
+ } else {
+ next = root;
+ }
+ }
+ if (next == null) {
+ throw new IllegalStateException("tree out of sync");
+ }
+ //Accept condition
+ if (next.hasMatch()) {
+ currentState.addResult(next.getMatchingTerm());
+ }
+ for (Node failNode = next.getFailureNode(); failNode != null; failNode = failNode.getFailureNode()) {
+ if (failNode.hasMatch()) {
+ currentState.addResult(failNode.getMatchingTerm());
+ }
+ }
+ current = next;
+ if (currentState.foundMatch() && !findAll) {
+ break;//give up as soon as we have at least one match
+ }
+ }
+ currentState.setCurrentNode(current);
+ return currentState;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
new file mode 100644
index 0000000..0ac325c
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search.ahocorasick;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.util.search.SearchTerm;
+
+/**
+ *
+ * @author
+ */
+public class Node {
+
+ private final Map<Integer, Node> neighborMap;
+ private Node failureNode;
+ private SearchTerm<?> term;
+
+ Node(final SearchTerm<?> term) {
+ this();
+ this.term = term;
+ }
+
+ Node() {
+ neighborMap = new HashMap<>();
+ term = null;
+ }
+
+ void setFailureNode(final Node fail) {
+ failureNode = fail;
+ }
+
+ public Node getFailureNode() {
+ return failureNode;
+ }
+
+ public boolean hasMatch() {
+ return term != null;
+ }
+
+ void setMatchingTerm(final SearchTerm<?> term) {
+ this.term = term;
+ }
+
+ public SearchTerm<?> getMatchingTerm() {
+ return term;
+ }
+
+ public Node getNeighbor(final int index) {
+ return neighborMap.get(index);
+ }
+
+ void setNeighbor(final Node neighbor, final int index) {
+ neighborMap.put(index, neighbor);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
new file mode 100644
index 0000000..6d36ad0
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search.ahocorasick;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.util.search.SearchTerm;
+
+public class SearchState<T> {
+
+ private Node currentNode;
+ private final Map<SearchTerm<T>, List<Long>> resultMap;
+ private long bytesRead;
+
+ SearchState(final Node rootNode) {
+ resultMap = new HashMap<>(5);
+ currentNode = rootNode;
+ bytesRead = 0L;
+ }
+
+ void incrementBytesRead(final long increment) {
+ bytesRead += increment;
+ }
+
+ void setCurrentNode(final Node curr) {
+ currentNode = curr;
+ }
+
+ public Node getCurrentNode() {
+ return currentNode;
+ }
+
+ public Map<SearchTerm<T>, List<Long>> getResults() {
+ return new HashMap<>(resultMap);
+ }
+
+ void addResult(final SearchTerm matchingTerm) {
+ final List<Long> indexes = (resultMap.containsKey(matchingTerm)) ? resultMap.get(matchingTerm) : new ArrayList<Long>(5);
+ indexes.add(bytesRead);
+ resultMap.put(matchingTerm, indexes);
+ }
+
+ public boolean foundMatch() {
+ return !resultMap.isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java b/commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
new file mode 100644
index 0000000..bd30a96
--- /dev/null
+++ b/commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import org.apache.nifi.remote.io.CompressionInputStream;
+import org.apache.nifi.remote.io.CompressionOutputStream;
+
+import org.junit.Test;
+
+public class TestCompressionInputOutputStreams {
+
+ @Test
+ public void testSimple() throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final byte[] data = "Hello, World!".getBytes("UTF-8");
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos);
+ cos.write(data);
+ cos.flush();
+ cos.close();
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
+ final byte[] decompressed = readFully(cis);
+
+ assertTrue(Arrays.equals(data, decompressed));
+ }
+
+ @Test
+ public void testDataLargerThanBuffer() throws IOException {
+ final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
+
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 100; i++) {
+ sb.append(str);
+ }
+ final byte[] data = sb.toString().getBytes("UTF-8");
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
+ cos.write(data);
+ cos.flush();
+ cos.close();
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
+ final byte[] decompressed = readFully(cis);
+
+ assertTrue(Arrays.equals(data, decompressed));
+ }
+
+ @Test
+ public void testDataLargerThanBufferWhileFlushing() throws IOException {
+ final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
+ final byte[] data = str.getBytes("UTF-8");
+
+ final StringBuilder sb = new StringBuilder();
+ final byte[] data1024;
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
+ for (int i = 0; i < 1024; i++) {
+ cos.write(data);
+ cos.flush();
+ sb.append(str);
+ }
+ cos.close();
+ data1024 = sb.toString().getBytes("UTF-8");
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
+ final byte[] decompressed = readFully(cis);
+
+ assertTrue(Arrays.equals(data1024, decompressed));
+ }
+
+ @Test
+ public void testSendingMultipleFilesBackToBackOnSameStream() throws IOException {
+ final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
+ final byte[] data = str.getBytes("UTF-8");
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
+ for (int i = 0; i < 512; i++) {
+ cos.write(data);
+ cos.flush();
+ }
+ cos.close();
+
+ final CompressionOutputStream cos2 = new CompressionOutputStream(baos, 8192);
+ for (int i = 0; i < 512; i++) {
+ cos2.write(data);
+ cos2.flush();
+ }
+ cos2.close();
+
+ final byte[] data512;
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 512; i++) {
+ sb.append(str);
+ }
+ data512 = sb.toString().getBytes("UTF-8");
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final ByteArrayInputStream bais = new ByteArrayInputStream(compressedBytes);
+
+ final CompressionInputStream cis = new CompressionInputStream(bais);
+ final byte[] decompressed = readFully(cis);
+ assertTrue(Arrays.equals(data512, decompressed));
+
+ final CompressionInputStream cis2 = new CompressionInputStream(bais);
+ final byte[] decompressed2 = readFully(cis2);
+ assertTrue(Arrays.equals(data512, decompressed2));
+ }
+
+ private byte[] readFully(final InputStream in) throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final byte[] buffer = new byte[65536];
+ int len;
+ while ((len = in.read(buffer)) >= 0) {
+ baos.write(buffer, 0, len);
+ }
+
+ return baos.toByteArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java b/commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
new file mode 100644
index 0000000..52bd8de
--- /dev/null
+++ b/commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import org.apache.nifi.stream.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.LeakyBucketStreamThrottler;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("Tests are time-based")
+public class TestLeakyBucketThrottler {
+
+ @Test(timeout = 10000)
+ public void testOutputStreamInterface() throws IOException {
+ // throttle rate at 1 MB/sec
+ final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+
+ final byte[] data = new byte[1024 * 1024 * 4];
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final OutputStream throttledOut = throttler.newThrottledOutputStream(baos);
+
+ final long start = System.currentTimeMillis();
+ throttledOut.write(data);
+ throttler.close();
+ final long millis = System.currentTimeMillis() - start;
+ // should take 4 sec give or take
+ assertTrue(millis > 3000);
+ assertTrue(millis < 6000);
+ }
+
+ @Test(timeout = 10000)
+ public void testInputStreamInterface() throws IOException {
+ // throttle rate at 1 MB/sec
+ final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+
+ final byte[] data = new byte[1024 * 1024 * 4];
+ final ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ final InputStream throttledIn = throttler.newThrottledInputStream(bais);
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final byte[] buffer = new byte[4096];
+ final long start = System.currentTimeMillis();
+ int len;
+ while ((len = throttledIn.read(buffer)) > 0) {
+ baos.write(buffer, 0, len);
+ }
+ throttler.close();
+ final long millis = System.currentTimeMillis() - start;
+ // should take 4 sec give or take
+ assertTrue(millis > 3000);
+ assertTrue(millis < 6000);
+ baos.close();
+ }
+
+ @Test(timeout = 10000)
+ public void testDirectInterface() throws IOException, InterruptedException {
+ // throttle rate at 1 MB/sec
+ final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+
+ // create 3 threads, each sending ~2 MB
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final List<Thread> threads = new ArrayList<Thread>();
+ for (int i = 0; i < 3; i++) {
+ final Thread t = new WriterThread(i, throttler, baos);
+ threads.add(t);
+ }
+
+ final long start = System.currentTimeMillis();
+ for (final Thread t : threads) {
+ t.start();
+ }
+
+ for (final Thread t : threads) {
+ t.join();
+ }
+ final long elapsed = System.currentTimeMillis() - start;
+
+ throttler.close();
+
+ // To send 15 MB, it should have taken at least 5 seconds and no more than 7 seconds, to
+ // allow for busy-ness and the fact that we could write a tiny bit more than the limit.
+ assertTrue(elapsed > 5000);
+ assertTrue(elapsed < 7000);
+
+ // ensure bytes were copied out appropriately
+ assertEquals(3 * (2 * 1024 * 1024 + 1), baos.getBufferLength());
+ assertEquals((byte) 'A', baos.getUnderlyingBuffer()[baos.getBufferLength() - 1]);
+ }
+
+ private static class WriterThread extends Thread {
+
+ private final int idx;
+ private final byte[] data = new byte[1024 * 1024 * 2 + 1];
+ private final LeakyBucketStreamThrottler throttler;
+ private final OutputStream out;
+
+ public WriterThread(final int idx, final LeakyBucketStreamThrottler throttler, final OutputStream out) {
+ this.idx = idx;
+ this.throttler = throttler;
+ this.out = out;
+ this.data[this.data.length - 1] = (byte) 'A';
+ }
+
+ @Override
+ public void run() {
+ long startMillis = System.currentTimeMillis();
+ long bytesWritten = 0L;
+ try {
+ throttler.copy(new ByteArrayInputStream(data), out);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return;
+ }
+ long now = System.currentTimeMillis();
+ long millisElapsed = now - startMillis;
+ bytesWritten += data.length;
+ float bytesPerSec = (float) bytesWritten / (float) millisElapsed * 1000F;
+ System.out.println(idx + " : copied data at a rate of " + bytesPerSec + " bytes/sec");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java b/commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
new file mode 100644
index 0000000..0838e96
--- /dev/null
+++ b/commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class TestNaiveSearchRingBuffer {
+
+ @Test
+ public void testAddAndCompare() {
+ final byte[] pattern = new byte[]{
+ '\r', '0', 38, 48
+ };
+
+ final byte[] search = new byte[]{
+ '\r', '0', 38, 58, 58, 83, 78, '\r', '0', 38, 48, 83, 92, 78, 4, 38
+ };
+
+ final NaiveSearchRingBuffer circ = new NaiveSearchRingBuffer(pattern);
+ int counter = -1;
+ for (final byte b : search) {
+ counter++;
+ final boolean matched = circ.addAndCompare(b);
+ if (counter == 10) {
+ assertTrue(matched);
+ } else {
+ assertFalse(matched);
+ }
+ }
+ }
+
+ @Test
+ public void testGetOldestByte() {
+ final byte[] pattern = new byte[]{
+ '\r', '0', 38, 48
+ };
+
+ final byte[] search = new byte[]{
+ '\r', '0', 38, 58, 58, 83, 78, (byte) 223, (byte) 227, (byte) 250, '\r', '0', 38, 48, 83, 92, 78, 4, 38
+ };
+
+ final NaiveSearchRingBuffer circ = new NaiveSearchRingBuffer(pattern);
+ int counter = -1;
+ for (final byte b : search) {
+ counter++;
+ final boolean matched = circ.addAndCompare(b);
+ if (counter == 13) {
+ assertTrue(matched);
+ } else {
+ assertFalse(matched);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java b/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java
new file mode 100644
index 0000000..f576e94
--- /dev/null
+++ b/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.util.UUID;
+
+
+import org.junit.Test;
+
+public class TestCompoundUpdateMonitor {
+
+ @Test
+ public void test() throws IOException {
+ final UpdateMonitor lastModified = new LastModifiedMonitor();
+ final MD5SumMonitor md5 = new MD5SumMonitor();
+ final CompoundUpdateMonitor compound = new CompoundUpdateMonitor(lastModified, md5);
+
+ final File file = new File("target/" + UUID.randomUUID().toString());
+ if (file.exists()) {
+ assertTrue(file.delete());
+ }
+ assertTrue(file.createNewFile());
+
+ final Path path = file.toPath();
+
+ final Object curState = compound.getCurrentState(path);
+ final Object state2 = compound.getCurrentState(path);
+
+ assertEquals(curState, state2);
+ file.setLastModified(System.currentTimeMillis() + 1000L);
+ final Object state3 = compound.getCurrentState(path);
+ assertEquals(state2, state3);
+
+ final Object state4 = compound.getCurrentState(path);
+ assertEquals(state3, state4);
+
+ final long lastModifiedDate = file.lastModified();
+ try (final OutputStream out = new FileOutputStream(file)) {
+ out.write("Hello".getBytes("UTF-8"));
+ }
+
+ file.setLastModified(lastModifiedDate);
+
+ final Object state5 = compound.getCurrentState(path);
+ assertNotSame(state4, state5);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java b/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java
new file mode 100644
index 0000000..7125581
--- /dev/null
+++ b/commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import org.junit.Test;
+
+
+public class TestSynchronousFileWatcher {
+
+ @Test
+ public void testIt() throws UnsupportedEncodingException, IOException, InterruptedException {
+ final Path path = Paths.get("target/1.txt");
+ Files.copy(new ByteArrayInputStream("Hello, World!".getBytes("UTF-8")), path, StandardCopyOption.REPLACE_EXISTING);
+ final UpdateMonitor monitor = new MD5SumMonitor();
+
+ final SynchronousFileWatcher watcher = new SynchronousFileWatcher(path, monitor, 10L);
+ assertFalse(watcher.checkAndReset());
+ Thread.sleep(30L);
+ assertFalse(watcher.checkAndReset());
+
+ final FileOutputStream fos = new FileOutputStream(path.toFile());
+ try {
+ fos.write("Good-bye, World!".getBytes("UTF-8"));
+ fos.getFD().sync();
+ } finally {
+ fos.close();
+ }
+
+ assertTrue(watcher.checkAndReset());
+ assertFalse(watcher.checkAndReset());
+
+ Thread.sleep(30L);
+ assertFalse(watcher.checkAndReset());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestCompoundUpdateMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestCompoundUpdateMonitor.java b/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestCompoundUpdateMonitor.java
deleted file mode 100644
index c796a96..0000000
--- a/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestCompoundUpdateMonitor.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.timebuffer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.file.Path;
-import java.util.UUID;
-
-import org.apache.nifi.io.CompoundUpdateMonitor;
-import org.apache.nifi.io.LastModifiedMonitor;
-import org.apache.nifi.io.MD5SumMonitor;
-import org.apache.nifi.io.UpdateMonitor;
-
-import org.junit.Test;
-
-public class TestCompoundUpdateMonitor {
-
- @Test
- public void test() throws IOException {
- final UpdateMonitor lastModified = new LastModifiedMonitor();
- final MD5SumMonitor md5 = new MD5SumMonitor();
- final CompoundUpdateMonitor compound = new CompoundUpdateMonitor(lastModified, md5);
-
- final File file = new File("target/" + UUID.randomUUID().toString());
- if (file.exists()) {
- assertTrue(file.delete());
- }
- assertTrue(file.createNewFile());
-
- final Path path = file.toPath();
-
- final Object curState = compound.getCurrentState(path);
- final Object state2 = compound.getCurrentState(path);
-
- assertEquals(curState, state2);
- file.setLastModified(System.currentTimeMillis() + 1000L);
- final Object state3 = compound.getCurrentState(path);
- assertEquals(state2, state3);
-
- final Object state4 = compound.getCurrentState(path);
- assertEquals(state3, state4);
-
- final long lastModifiedDate = file.lastModified();
- try (final OutputStream out = new FileOutputStream(file)) {
- out.write("Hello".getBytes("UTF-8"));
- }
-
- file.setLastModified(lastModifiedDate);
-
- final Object state5 = compound.getCurrentState(path);
- assertNotSame(state4, state5);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestSynchronousFileWatcher.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestSynchronousFileWatcher.java b/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestSynchronousFileWatcher.java
deleted file mode 100644
index 4b2c0d5..0000000
--- a/commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestSynchronousFileWatcher.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.timebuffer;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-
-import org.junit.Test;
-
-import org.apache.nifi.io.MD5SumMonitor;
-import org.apache.nifi.io.SynchronousFileWatcher;
-import org.apache.nifi.io.UpdateMonitor;
-
-public class TestSynchronousFileWatcher {
-
- @Test
- public void testIt() throws UnsupportedEncodingException, IOException, InterruptedException {
- final Path path = Paths.get("target/1.txt");
- Files.copy(new ByteArrayInputStream("Hello, World!".getBytes("UTF-8")), path, StandardCopyOption.REPLACE_EXISTING);
- final UpdateMonitor monitor = new MD5SumMonitor();
-
- final SynchronousFileWatcher watcher = new SynchronousFileWatcher(path, monitor, 10L);
- assertFalse(watcher.checkAndReset());
- Thread.sleep(30L);
- assertFalse(watcher.checkAndReset());
-
- final FileOutputStream fos = new FileOutputStream(path.toFile());
- try {
- fos.write("Good-bye, World!".getBytes("UTF-8"));
- fos.getFD().sync();
- } finally {
- fos.close();
- }
-
- assertTrue(watcher.checkAndReset());
- assertFalse(watcher.checkAndReset());
-
- Thread.sleep(30L);
- assertFalse(watcher.checkAndReset());
- }
-}
[13/31] incubator-nifi git commit: NIFI-186: Added additional args to
bootstrap conf file
Posted by mc...@apache.org.
NIFI-186: Added additional args to bootstrap conf file
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/b0a51150
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/b0a51150
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/b0a51150
Branch: refs/heads/NIFI-182
Commit: b0a5115097eb1fb30a390c0dc5da64edb60ed2dc
Parents: f048ed7
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 14:25:26 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 14:25:26 2014 -0500
----------------------------------------------------------------------
.../src/main/resources/conf/bootstrap.conf | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b0a51150/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf b/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
index 37ec474..6208560 100644
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
+++ b/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
@@ -1,3 +1,9 @@
+# Java command to use when running NiFi
+java=java
+
+# Username to use when running NiFi. This value will be ignored on Windows.
+run.as=
+
# Configure where NiFi's lib and conf directories live
lib.dir=./lib
conf.dir=./conf
@@ -15,8 +21,14 @@ java.arg.3=-Xmx512m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
-# Java command to use when running NiFi
-java=java
+java.arg.4=-Djava.net.preferIPv4Stack=true
-# Username to use when running NiFi. This value will be ignored on Windows.
-run.as=
+# allowRestrictedHeaders is required for Cluster/Node communications to work properly
+java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
+java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
+
+# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with
+# many classes loaded in the JVM.
+java.arg.7=-XX:+ReservedCodeCacheSize=256m
+java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m
+java.arg.9=-XX:+UseCodeCacheFlushing
[02/31] incubator-nifi git commit: NIFI-150: Added documentation in
asciidoc format
Posted by mc...@apache.org.
NIFI-150: Added documentation in asciidoc format
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/a23c7f2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/a23c7f2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/a23c7f2e
Branch: refs/heads/NIFI-182
Commit: a23c7f2e2b6b57adf7adbf00e81d05ec1d657bb2
Parents: 578974e
Author: Mark Payne <ma...@hotmail.com>
Authored: Thu Dec 18 17:10:42 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Thu Dec 18 17:10:42 2014 -0500
----------------------------------------------------------------------
nifi-docs/pom.xml | 38 ++
.../images/add-processor-with-tag-cloud.png | Bin 0 -> 22925 bytes
.../src/main/asciidoc/images/add-processor.png | Bin 0 -> 31524 bytes
.../src/main/asciidoc/images/addConnect.png | Bin 0 -> 1996 bytes
.../src/main/asciidoc/images/comments-tab.png | Bin 0 -> 6431 bytes
.../src/main/asciidoc/images/components.png | Bin 0 -> 8992 bytes
.../asciidoc/images/connection-settings.png | Bin 0 -> 15214 bytes
.../main/asciidoc/images/create-connection.png | Bin 0 -> 9430 bytes
.../asciidoc/images/edit-property-dropdown.png | Bin 0 -> 14291 bytes
.../asciidoc/images/edit-property-textarea.png | Bin 0 -> 17146 bytes
.../src/main/asciidoc/images/iconConnection.png | Bin 0 -> 1517 bytes
.../src/main/asciidoc/images/iconDelete.png | Bin 0 -> 670 bytes
.../src/main/asciidoc/images/iconFunnel.png | Bin 0 -> 1223 bytes
nifi-docs/src/main/asciidoc/images/iconInfo.png | Bin 0 -> 550 bytes
.../src/main/asciidoc/images/iconInputPort.png | Bin 0 -> 1842 bytes
.../src/main/asciidoc/images/iconLabel.png | Bin 0 -> 838 bytes
.../src/main/asciidoc/images/iconOutputPort.png | Bin 0 -> 1658 bytes
.../main/asciidoc/images/iconProcessGroup.png | Bin 0 -> 1422 bytes
.../src/main/asciidoc/images/iconProcessor.png | Bin 0 -> 1446 bytes
.../asciidoc/images/iconRemoteProcessGroup.png | Bin 0 -> 674 bytes
.../src/main/asciidoc/images/iconTemplate.png | Bin 0 -> 970 bytes
.../images/instantiate-template-description.png | Bin 0 -> 6508 bytes
.../asciidoc/images/instantiate-template.png | Bin 0 -> 3216 bytes
nifi-docs/src/main/asciidoc/images/new-flow.png | Bin 0 -> 53989 bytes
.../main/asciidoc/images/nifi-navigation.png | Bin 0 -> 85458 bytes
.../asciidoc/images/nifi-toolbar-components.png | Bin 0 -> 72263 bytes
.../images/processor-connection-bubble.png | Bin 0 -> 9936 bytes
.../src/main/asciidoc/images/properties-tab.png | Bin 0 -> 11847 bytes
.../src/main/asciidoc/images/scheduling-tab.png | Bin 0 -> 12288 bytes
.../src/main/asciidoc/images/settings-tab.png | Bin 0 -> 16225 bytes
.../src/main/asciidoc/images/status-bar.png | Bin 0 -> 57171 bytes
.../src/main/asciidoc/nifi-user-guide.adoc | 519 +++++++++++++++++++
pom.xml | 8 +-
33 files changed, 561 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
new file mode 100644
index 0000000..1b74533
--- /dev/null
+++ b/nifi-docs/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-docs</artifactId>
+
+ <name>nifi-docs</name>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.asciidoctor</groupId>
+ <artifactId>asciidoctor-maven-plugin</artifactId>
+ <version>1.5.0</version>
+ <executions>
+ <execution>
+ <id>output-html</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>process-asciidoc</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <backend>html</backend>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png b/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png
new file mode 100644
index 0000000..a0bbadb
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/add-processor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/add-processor.png b/nifi-docs/src/main/asciidoc/images/add-processor.png
new file mode 100644
index 0000000..a043f38
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/add-processor.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/addConnect.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/addConnect.png b/nifi-docs/src/main/asciidoc/images/addConnect.png
new file mode 100644
index 0000000..b33a305
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/addConnect.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/comments-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/comments-tab.png b/nifi-docs/src/main/asciidoc/images/comments-tab.png
new file mode 100644
index 0000000..b6ed34b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/comments-tab.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/components.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/components.png b/nifi-docs/src/main/asciidoc/images/components.png
new file mode 100644
index 0000000..b3a9769
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/components.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/connection-settings.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/connection-settings.png b/nifi-docs/src/main/asciidoc/images/connection-settings.png
new file mode 100644
index 0000000..9aae93a
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/connection-settings.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/create-connection.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/create-connection.png b/nifi-docs/src/main/asciidoc/images/create-connection.png
new file mode 100644
index 0000000..7ca40ae
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/create-connection.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png b/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png
new file mode 100644
index 0000000..017805b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png b/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png
new file mode 100644
index 0000000..cc8668a
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconConnection.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconConnection.png b/nifi-docs/src/main/asciidoc/images/iconConnection.png
new file mode 100644
index 0000000..4b6180b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconConnection.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconDelete.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconDelete.png b/nifi-docs/src/main/asciidoc/images/iconDelete.png
new file mode 100644
index 0000000..dc13227
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconDelete.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconFunnel.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconFunnel.png b/nifi-docs/src/main/asciidoc/images/iconFunnel.png
new file mode 100644
index 0000000..98f9b9d
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconFunnel.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconInfo.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconInfo.png b/nifi-docs/src/main/asciidoc/images/iconInfo.png
new file mode 100644
index 0000000..f192ff5
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconInfo.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconInputPort.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconInputPort.png b/nifi-docs/src/main/asciidoc/images/iconInputPort.png
new file mode 100644
index 0000000..d4efd97
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconInputPort.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconLabel.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconLabel.png b/nifi-docs/src/main/asciidoc/images/iconLabel.png
new file mode 100644
index 0000000..6ba3c13
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconLabel.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconOutputPort.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconOutputPort.png b/nifi-docs/src/main/asciidoc/images/iconOutputPort.png
new file mode 100644
index 0000000..f3d8a66
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconOutputPort.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png b/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png
new file mode 100644
index 0000000..c587341
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconProcessor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconProcessor.png b/nifi-docs/src/main/asciidoc/images/iconProcessor.png
new file mode 100644
index 0000000..2ca062c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconProcessor.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png b/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png
new file mode 100644
index 0000000..5205470
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/iconTemplate.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconTemplate.png b/nifi-docs/src/main/asciidoc/images/iconTemplate.png
new file mode 100644
index 0000000..66d2caa
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconTemplate.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png b/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png
new file mode 100644
index 0000000..a238148
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/instantiate-template.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/instantiate-template.png b/nifi-docs/src/main/asciidoc/images/instantiate-template.png
new file mode 100644
index 0000000..435d562
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/instantiate-template.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/new-flow.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/new-flow.png b/nifi-docs/src/main/asciidoc/images/new-flow.png
new file mode 100644
index 0000000..59f64d8
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/new-flow.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/nifi-navigation.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-navigation.png b/nifi-docs/src/main/asciidoc/images/nifi-navigation.png
new file mode 100644
index 0000000..1841cba
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-navigation.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png b/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png
new file mode 100644
index 0000000..3fd7170
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png b/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png
new file mode 100644
index 0000000..82a161b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/properties-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/properties-tab.png b/nifi-docs/src/main/asciidoc/images/properties-tab.png
new file mode 100644
index 0000000..0e60fb9
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/properties-tab.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/scheduling-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/scheduling-tab.png b/nifi-docs/src/main/asciidoc/images/scheduling-tab.png
new file mode 100644
index 0000000..20ca0f5
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/scheduling-tab.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/settings-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/settings-tab.png b/nifi-docs/src/main/asciidoc/images/settings-tab.png
new file mode 100644
index 0000000..77d6bd7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/settings-tab.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/images/status-bar.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/status-bar.png b/nifi-docs/src/main/asciidoc/images/status-bar.png
new file mode 100644
index 0000000..7030eca
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/status-bar.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
new file mode 100644
index 0000000..931819d
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
@@ -0,0 +1,519 @@
+NiFi User Guide
+===============
+Apache_NiFi_Team
+:toc:
+:icons:
+
+
+[template="glossary", id="terminology"]
+Terminology
+-----------
+*DataFlow Manager*: A DataFlow Manager (DFM) is a NiFi user who has permissions to add, remove, and modify components of a NiFi dataflow.
+
+*FlowFile*: The FlowFile represents a single piece of data in NiFi. A FlowFile is made up of two components:
+ FlowFile Attributes and FlowFile Content.
+ Content is the data that is represented by the FlowFile. Attributes are key-value pairs that provide information or
+ context about the data.
+ All FlowFiles have the following Standard Attributes:
+
+- *uuid*: A unique identifier for the FlowFile
+- *filename*: A human-readable filename that may be used when storing the data to disk or in an external service
+- *path*: A hierarchically structured value that can be used when storing data to disk or an external service so that the data is not stored in a single directory
+
+*Processor*: The Processor is the NiFi component that is used to listen for incoming data; pull data from external sources;
+ publish data to external sources; and route, transform, or extract information from FlowFiles.
+
+*Relationship*: Each Processor has zero or more Relationships defined for it. These Relationships are named to indicate the result of processing a FlowFile.
+ After a Processor has finished processing a FlowFile, it will route (or “transfer”) the FlowFile to one of the Relationships.
+ A DataFlow Manager is then able to connect each of these Relationships to other components in order to specify where the FlowFile should
+ go next under each potential processing result.
+
+*Connection*: A DataFlow Manager creates an automated dataflow by dragging components from the Components part of the NiFi toolbar to the canvas
+ and then connecting the components together via Connections. Each connection consists of one or more Relationships.
+ For each Connection that is drawn, a DataFlow Manager can determine which Relationships should be used for the Connection.
+ This allows data to be routed in different ways based on its processing outcome. Each connection houses a FlowFile Queue.
+ When a FlowFile is transferred to a particular Relationship, it is added to the queue belonging to the associated Connection.
+
+*Funnel*: A funnel is a NiFi component that is used to combine the data from several Connections into a single Connection.
+
+*Process Group*: When a dataflow becomes complex, it often is beneficial to reason about the dataflow at a higher, more abstract level.
+ NiFi allows multiple components, such as Processors, to be grouped together into a Process Group.
+ The NiFi User Interface then makes it easy for a DataFlow Manager to connect together multiple Process Groups into a logical dataflow,
+ as well as allowing the DataFlow Manager to enter a Process Group in order to see and manipulate the components within the Process Group.
+
+*Port*: Dataflows that are constructed using one or more Process Groups need a way to connect a Process Group to other dataflow components.
+ This is achieved by using Ports. A DataFlow Manager can add any number of Input Ports and Output Ports to a Process Group and name these ports appropriately.
+
+*Remote Process Group*: Just as data is transferred into and out of a Process Group, it is sometimes necessary to transfer data from one instance of NiFi to another.
+ While NiFi provides many different mechanisms for transferring data from one system to another, Remote Process Groups are often the easiest way to accomplish
+ this if transferring data to another instance of NiFi.
+
+*Bulletin*: The NiFi User Interface provides a significant amount of monitoring and feedback about the current status of the application.
+ In addition to rolling statistics and the current status that are provided for each component, components are able to report Bulletins.
+ Whenever a component reports a Bulletin, an icon is displayed on that component (or on the Status bar near the top of the page, for System-Level Bulletins).
+ Using the mouse to hover over that icon will provide a tool-tip that shows the time and severity (Debug, Info, Warning, Error) of the bulletin,
+ as well as the message of the Bulletin.
+ Bulletins from all components can also be viewed and filtered in the Bulletins Page, available in the Management Toolbar.
+
+*Template*: Often times, a dataflow is comprised of many sub-flows that could be reused. NiFi allows DataFlow Managers to select a part of the dataflow
+ (or the entire dataflow) and create a Template. This Template is given a name can then be dragged onto the canvas just like the other components.
+ As a result, several components be combined together to make a larger building block from which to create a dataflow.
+ These templates can also be exported as XML and imported into another NiFi instance, allowing these building blocks to be shared.
+
+
+
+NiFi User Interface
+-------------------
+
+The NiFi User Interface (UI) provides mechanisms for creating automated dataflows, as well as visualizing,
+editing, monitoring, and administering those dataflows. The UI can be broken down into several different segments,
+each responsible for different functionality of the application. We will begin by looking at screenshots of the
+application and labeling the different segments of the UI. We will provide a brief explanation of the purpose of each segment.
+Then, in the following sections of this document, we will discuss each of those segments in greater detail.
+
+When the application is started, by default, the user is able to navigate to the User Interface by going to
+`http://<hostname>:8080/nifi` in a web browser. There are no permissions configured, by default, so anyone is
+able to view and modify the dataflow. For information on securing the system, see Systems Administrator guide.
+
+When a DataFlow Manager navigates to the UI for the first time, a blank canvas is provided on which a dataflow can be built:
+
+image::new-flow.png["Empty Flow"]
+
+Along the top of the of the screen is a toolbar that contains several of these segments.
+To the left is the Components Toolbar. This toolbar consists of the different components that can be dragged onto the canvas.
+
+Next to the Components Toolbar is the Actions Toolbar. This toolbar consists of buttons to manipulate the existing
+components on the graph. Following the Actions Toolbar is the Search Toolbar. This toolbar consists of a single
+Search field that allows users to easily find components on the graph. Users are able to search by component name,
+type, identifier, and configuration properties.
+
+Finally, the Management Toolbar sits to the right-hand side of the screen. This toolbar consists of buttons that are
+of use to DataFlow Managers to manage the flow as well as administrators who may use this section to manage user access
+and configure system properties, such as how many system resources should be provided to the application.
+
+image::nifi-toolbar-components.png["NiFi Components Toolbar"]
+
+Next, we have segments that provide capabilities to easily navigate around the graph. On the left-hand side is a toolbar that
+provides the ability to pan around the graph and zoom in and out. On the right-hand side is a “Birds-Eye View” of the dataflow.
+This provides a high-level view of the dataflow and allows the user to quickly and easily pan across large portions of the dataflow.
+Along the top of the screen is a trail of breadcrumbs. As users navigate into and out of Process Groups, the breadcrumbs show
+the depth in the flow and each Process Group that was entered to reach this depth. Each of the Process Groups listed in the breadcrumbs
+is a link that will take you back up to that level in the flow.
+
+image::nifi-navigation.png["NiFi Navigation"]
+
+Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist in the graph in
+each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the graph in each state
+(Transmitting, Not Transmitting), the number of threads that are currently active in the flow, the amount of data that currently
+exists in the flow, and the timestamp at which all of this information was last refreshed. If there are any System-Level bulletins,
+these are shown in the Status bar as well. Additionally, if the instance of NiFi is clustered, the Status bar shows many nodes
+are in the cluster and how many are currently connected.
+
+image::status-bar.png["NiFi Status Bar"]
+
+
+
+
+
+Building a DataFlow
+-------------------
+
+A DataFlow Manager (DFM) is able to build an automated dataflow using the NiFi User Interface (UI). This is accomplished
+by dragging components from the toolbar to the canvas, configuring the components to meet specific needs, and connecting
+the components together.
+
+
+=== Adding Components to the Canvas
+
+In the User Interface section above, we outlined the different segments of the UI and pointed out a Components Toolbar.
+Here, we will look at each of the Components in that toolbar:
+
+image::components.png["Components"]
+
+image:iconProcessor.png["Processor", width=32]
+*Processor*: The Processor is the most commonly used component, as it is responsible for data ingress, egress, routing, and
+ manipulating. There are many different types of Processors. In fact, this is a very common Extension Point in NiFi,
+ meaning that many vendors may implement their own Processors to perform whatever functions are necessary for their use case.
+ When a Processor is dragged onto the graph, the user is presented with a dialog to choose which type of Processor to use:
+
+image::add-processor.png["Add Processor Dialog"]
+
+In the top-right corner, the user is able to filter the list based on the Processor Type or the Tags associated with a Processor.
+Processor developers have the ability to add Tags to their Processors. These tags are used in this dialog for filtering and are
+displayed on the left-hand side in a Tag Cloud. The more Processors that exist with a particular Tag, the larger the Tag appears
+in the Tag Cloud. Clicking a Tag in the Cloud will filter the available Processors to only those that contain that Tag. If multiple
+Tags are selected, only those Processors that contain all of those Tags are shown. For example, if we want to show only those
+Processors that allow us to ingest data via HTTP, we can select both the `http` Tag and the `ingest` Tag:
+
+image::add-processor-with-tag-cloud.png["Add Processor with Tag Cloud"]
+
+Clicking the `Add` button or double-clicking on a Processor Type will add the selected Processor to the canvas at the
+location that it was dropped.
+
+
+
+image:iconInputPort.png["Input Port", width=32]
+*Input Port*: Input Ports provide a mechanism for transferring data into a Process Group. When an Input Port is dragged
+onto the canvas, the DFM is prompted to name the Port. All Ports within a Process Group must have unique names.
+
+All components exist only within a Process Group. When a user navigates to the NiFi page, the user is placed in the
+Root Progress Group. If the Input Port is dragged onto the Root Progress Group, the Input Port provides a mechanism
+to receive data from remote instances of NiFi. In this case, the Input Port can be configured to restrict access to
+appropriate users.
+
+
+
+
+image:iconOutputPort.png["Output Port", width=32]
+*Output Port*: Output Ports provide a mechanism for transferring data from a Process Group back to destination outside
+of the Process Group. When an Output Port is dragged onto the canvas, the DFM is prompted to name the Port. All Ports
+within a Process Group must have unique names.
+
+If the Output Port is dragged onto the Root Process Group, the Output Port provides a mechanism for sending data to
+remote instances of NiFi. In this case, the Port acts as a queue. As remote instances of NiFi pull data from the port,
+that data is removed from the queues of the incoming Connections.
+
+
+
+image:iconProcessGroup.png["Process Group", width=32]
+*Process Group*: Process Groups can be used logically group a set of components so that the dataflow is easier to understand
+and maintain. When a Process Group is dragged onto the canvas, the DFM is prompted to name the Process Group. All Process
+Groups within the same parent group must have unique names.
+
+
+
+image:iconRemoteProcessGroup.png["Remote Process Group", width=32]
+*Remote Process Group*: Remote Process Groups appear and behave similar to Process Groups. However, the Remote Process Group (RPG)
+references a remote instance of NiFi. When an RPG is dragged onto the canvas, rather than being prompted for a name, the DFM
+is prompted for the URL of the remote NiFi instance. If the remote NiFi is a clustered instance, the URL that should be used
+is the URL of the remote instance's NiFi Cluster Manager (NCM). When data is transferred to a clustered instance of NiFi
+via an RPG, the RPG it will first connect to the remote instance's NCM to determine which nodes are in the cluster and
+how busy each node is. This information is then used to load balance the data that is pushed to each node. The remote NCM is
+then interrogated periodically to ensure that any nodes that are dropped from the cluster and no longer sent to, any new nodes
+will be added to the list of nodes, and to recalculate the load balancing based on each node's load.
+
+
+
+image:iconFunnel.png["Funnel", width=32]
+*Funnel*: Funnels are used to combine the data from many Connections into a single Connection. This has two advantages.
+First, if many Connections are created with the same destination, the canvas can become cluttered if those Connections
+have to span a large space. By funneling these Connections into a single Connection, that single Connection can then be
+drawn to span that large space instead. Secondly, Connections can be configured with FlowFile Prioritizers. Data from
+several Connections can be funneled into a single Connection, providing the ability to Prioritize all of the data on that
+one Connection, rather than prioritizing the data on each Connection independently.
+
+
+
+image:iconTemplate.png["Template", width=32]
+*Template*: Templates can be created by DataFlow Managers from sections of the flow, or they can be imported from other
+dataflows. These Templates provide larger building blocks for creating a complex flow quickly. When the Template is
+dragged onto the canvas, the DFM is provided a dialog to choose which Template to add to the canvas:
+
+image::instantiate-template.png["Instantiate Template Dialog"]
+
+Clicking the drop-down box shows all available Templates. Any Template that was created with a description will show an
+icon indicating that there is more information. Hovering over the icon with the mouse will show this description:
+
+image::instantiate-template-description.png["Instantiate Template Dialog"]
+
+
+
+
+image:iconLabel.png["Label"]
+*Label*: Labels are used to provide documentation to parts of a dataflow. When a Label is dropped onto the canvas,
+it is created with a default size. The Label can then be resized by dragging the handle in the bottom-right corner.
+The Label has no text when initially created. The text of the Label can be added by right-clicking on the Label and
+choosing `Configure...`
+
+
+
+
+=== Configuring a Processor
+
+Once a Processor has been dragged onto the Canvas, it is ready to configure. This is done by right-clicking on the
+Processor and clicking the `Configure...` option from the context menu. The configuration dialog is opened with four
+different tabs, each of which is discussed below. Once you have finished configuring the Processor, you can apply
+the changes by clicking the `Apply` button or cancel all changes by clicking the `Cancel` button.
+
+Note that after a Processor has been started, the context menu shown for the Processor no longer has a `Configure...`
+option but rather has a `View Configuration` option. Processor configuration cannot be changed while the Processor is
+running. You must first stop the Processor and wait for all of its active tasks to complete before configuring
+the Processor again.
+
+
+==== Settings Tab
+
+The first tab in the Processor Configuration dialog is the Settings tab:
+
+image::settings-tab.png["Settings Tab"]
+
+This tab contains several different configuration items. First, it allows the DFM to change the name of the Processor.
+The name of a Processor by default is the same as the Processor type. Next to the Processor Name is a control for
+determining whether or not the Processor is Enabled. When a Processor is added to the graph, it is enabled. If the
+Processor is disabled, it cannot be started. This is used to indicate that even when a group of Processors are started,
+such as when a DFM starts an entire Process Group, this Processor should be excluded.
+
+Below the Name configuration, the Processor's unique identifier is displayed along with the Processor's type. These
+values cannot be modified.
+
+Next are two dialogues for configuring `Penalty duration' and `Yield duration'. During the normal course of processing a
+piece of data (a FlowFile), an event may occur that indicates that the data cannot be processed at this time but the
+data may be processable at a later time. When this occurs, the Processor may choose to Penalize the FlowFile. This will
+prevent the FlowFile from being Processed for some period of time. For example, if the Processor is to push the data
+to a remote service, but the remote service already has a file with the same name as the filename that the Processor
+is specifying, the Processor may penalize the FlowFile. The `Penalty duration' allows the DFM to specify what
+how long the FlowFile should be penalized. The default value is 30 seconds.
+
+Similarly, the Processor may determine that some situation exists such that the Processor can no longer make any progress,
+regardless of the data that it is processing. For example, if a Processor is to push data to a remote service and that
+service is not responding, the Processor cannot make any progress. As a result, the Processor should `yield,' which will
+prevent the Processor from being scheduled to run for some period of time. That period of time is specified by setting
+the `Yield duration.' The default value is 1 second.
+
+The last configurable option on the left-hand side of the Settings tab is the Bulletin level. Whenever the Processor writes
+to its log, the Processor also will generate a Bulletin. This setting indicates the lowest level of Bulletin that should be
+shown in the User Interface. By default, the Bulletin level is set to WARN.
+
+The right-hand side of the dialogue provides an `Auto-terminate relationships' section. Each of the Relationships that is
+defined by the Processor is listed here, along with its description. In order for a Processor to be considered valid and
+able to run, each Relationship defined by the Processor must be either connected to a downstream component or auto-terminated.
+If a Relationship is auto-terminated, any FlowFile that is routed to that Relationship will be removed from the flow and
+its processing considered complete. Any Relationship that is already connected to a downstream component cannot be auto-terminated.
+The Relationship must first be removed from any Connection that uses it. Additionally, for any Relationship that is selected to be
+auto-terminated, the auto-termination status will be cleared if the Relationship is added to a Connection.
+
+
+
+
+==== Scheduling Tab
+
+The second tab in the Processor Configuration dialog is the Scheduling Tab:
+
+image::scheduling-tab.png["Scheduling Tab"]
+
+The first configuration option is the Scheduling Strategy. There are three options for scheduling components:
+
+- *Timer driven*: This is the default mode. The Processor will be scheduled to run on a regular interval. The interval
+ at which the Processor is run is defined by the `Run schedule' option (see below).
+- *Event driven*: When this mode is selected, the Processor will be triggered to run by FlowFiles entering the Connections
+ that have this Processor as their destination. This mode is not supported by all Processors. When this mode is
+ selected, the `Run schedule' option is not configurable, as the Processor is not triggered to run periodically but
+ rather is triggered to run as the result of an event. Additionally, this is the only mode for which the `Concurrent tasks'
+ option can be set to 0. In this case, the number of threads is limited only by the size of the Event-Driven Thread Pool that
+ the administrator has configured.
+- *CRON driven*: When using the CRON driven scheduling mode, the Processor is scheduled to run periodically, similarly to the
+ Timer driven scheduling mode. However, the CRON driven mode provides significantly more flexibility at the expensive of
+ increasing the complexity of the configuration. This value is made up of 6 fields, each separated by a space. These
+ fields represent the following fields:
++
+ . Seconds
+ . Minutes
+ . Hours
+ . Day of Month
+ . Month
+ . Day of Week
+ . Year
++
+The value for each of these fields should be a number, range, or increment.
+Range here refers to a syntax of <number>-<number>.
+For example,the Seconds field could be set to 0-30, meaning that the Processor should only be scheduled if the time is 0 to 30 seconds
+after the minute. Additionally, a value of `*` indicates that all values are valid for this field. Multiple values can also
+be entered using a `,` as a separator: `0,5,10,15,30`.
+An increment is written as <start value>/<increment>. For example, settings a value of `0/10` for the seconds fields means that valid
+values are 0, 10, 20, 30, 40, and 50. However, if we change this to `5/10`, valid values become 5, 15, 25, 35, 45, and 55.
++
+For the Month field, valid values are 1 (January) through 12 (December).
++
+For the Day of Week field, valid values are 1 (Sunday) through 7 (Saturday). Additionally, a value of `L` may be appended to one of these
+values to indicate the last occurrence of this day in the month. For example, `1L` can be used to indicate the last Monday of the month.
+
+
+Next, the Scheduling Tab provides a configuration option named `Concurrent tasks.' This controls how many threads the Processor
+will use. Said a different way, this controls how many FlowFiles should be processed by this Processor at the same time. Increasing
+this value will typically allow the Processor to handle more data in the same amount of time. However, it does this by using system
+resources that then are not usable by other Processors. This essentially provides a relative weighting of Processors -- it controls
+how much of the system's resources should be allocated to this Processor instead of other Processors. This field is available for
+most Processors. There are, however, some types of Processors that can only be scheduled with a single Concurrent task.
+
+The `Run schedule` dictates how often this Processor should be scheduled to run. The valid values for this field depend on the selected
+Scheduling Strategy (see above). If using the Event driven Scheduling Strategy, this field is not available. When using the Timer driven
+Scheduling Strategy, this value is a time duration specified by a number followed by a time unit. For example, `1 second` or `5 mins`.
+The default value of `0 sec` means that the Processor should run as often as possible as long as it has data to process. This is true
+for any time duration of 0, regardless of the time unit (i.e., `0 sec`, `0 mins`, `0 days`). For an explanation of values that are
+applicable for the CRON driven Scheduling Strategy, see the description of the CRON driven Scheduling Strategy itself.
+
+The right-hand side of the tab contains a slider for choosing the `Run duration.' This controls how long the Processor should be scheduled
+to run each time that it is triggered. On the left-hand side of the slider, it is marked `Lower latency' while the right-hand side
+is marked `Higher throughput.' When a Processor finishes running, it must update the repository in order to transfer the FlowFiles to
+the next Connection. Updating this repository is expensive, so the more work that can be done at once before updating the repository
+the more work the Processor can handle (Higher throughput). However, this means that the next Processor cannot start processing
+those FlowFiles until the previous Process updates this repository. As a result, the latency will be longer (the time required to process
+the FlowFile from beginning to end will be longer). As a result, the slider provides a spectrum from which the DFM can choose to favor
+Lower Latency or Higher Throughput.
+
+
+==== Properties Tab
+
+The Properties Tab provides a mechanism to configure Processor-specific behavior. There are no default properties. Each type of Processor
+must define which Properties make sense for its use case. Below, we see the Properties Tab for a RouteOnAttribute Processor:
+
+image::properties-tab.png["Properties Tab"]
+
+This Processor, by default, has only a single property: `Routing Strategy.' The default value is `Route on Property name.' Next to
+the name of this property is a small question-mark symbol (
+image:iconInfo.png["Question Mark"]
+). This help symbol is seen in other places throughout the application, as well, and indicates that more information is available.
+Hovering over this symbol with the mouse will provide additional details about the property and the default value, as well as
+historical values that have been set for the Property.
+
+Clicking on the value for the property will allow a DFM to change the value. Depending on the values that are allowed for the property,
+the user is either provided a drop-down from which to choose a value or is given a text area to type a value:
+
+image::edit-property-dropdown.png["Edit Property with Dropdown"]
+
+In the top-right corner of the tab is a button for adding a New Property. Clicking this button will provide the DFM with a dialog to
+enter the name and value of a new property. Not all Processors allow User-Defined properties. In this case, the Processor would become
+invalid when the properties are applied. RouteOnAttribute, for example, does allow User-Defined properties. In fact, this Processor
+will not be valid until the user has added a property.
+
+image:edit-property-textarea.png["Edit Property with Text Area"]
+
+Not that after a User-Defined property has been added, an icon will appear on the right-hand side of that row (
+image:iconDelete.png["Delete Icon"]
+). Clicking this button will remove the User-Defined property from the Processor.
+
+
+
+==== Comments Tab
+
+The last tab in the Processor configuration dialog is the Comments tab. This tab simply provides an area for users to provide
+whatever comments are appropriate for this component:
+
+image::comments-tab.png["Comments Tab"]
+
+
+=== Additional Help
+
+
+
+=== Connecting Components
+
+After the appropriate Processors have been added to the graph and configured to meet your needs, they will have to be connected
+to one another so that NiFi knows what to do with each FlowFile after it has been processed. This is accomplished by creating a
+Connection between two components. When the mouse hovers over a component, a new Connection icon (
+image:addConnect.png["Connection Bubble"]
+) will appear in the middle of the component:
+
+image:processor-connection-bubble.png["Processor with Connection Bubble"]
+
+This Connection bubble can then be dragged from this component to another component, which will provide to the user a
+`Create Connection' dialog. This dialog consists of two tabs: `Details' and `Settings'.
+
+==== Details Tab
+
+The Details Tab provides information about the source and destination components, including the component name, the
+component type, and the Process Group in which the component lives:
+
+image::create-connection.png["Create Connection"]
+
+Additionally, this tab provides the ability to choose which Relationships should be included in this Connection. At least one
+Relationship must be selected. If only one Relationship is available, it is automatically selected.
+
+*Note*: If multiple Connections are added with the same Relationship, any FlowFile that is routed to that Relationship will
+automatically be `cloned', and a copy will be sent to each of those Connections.
+
+==== Settings
+
+The Settings Tab provides the ability to configure the Connection's name, FlowFile expiration, back pressure thresholds, and
+Prioritization:
+
+image:connection-settings.png["Connection Settings"]
+
+The Connection name is optional. If not specified, the name shown for the Connection will be names of the Relationships
+that are active for the Connection.
+
+File expiration is a concept by which data that cannot be processed in a timely fashion can be automatically destroyed.
+This is useful, for example, when the volume of data is expected to exceed the volume that can be sent to a remote site.
+In this case, the expiration can be used in conjunction with Prioritizers to ensure that the highest priority data is
+processed first and then anything that cannot be processed within one hour, for example, can be dropped. The default
+value of `0 sec` indicates that the data will never expire.
+
+NiFi provides two different configuration elements for back pressure. These thresholds indicate how much data should be
+allowed to exist in the queue before the component that is the source of the Connection is no longer scheduled to run.
+This allows the system to avoid being overrun with data.
+
+
+
+
+=== Processor Validation
+
+
+
+
+
+
+
+== Command and Control of DataFlow
+
+=== Starting a Component
+
+=== Stopping a Component
+
+=== Enabling a Component
+
+=== Disabling a Component
+
+
+
+
+
+
+
+== Templates
+
+=== Creating a Template
+
+=== Instantiating a Template
+
+=== Managing Templates
+==== Importing a Template
+==== Exporting a Template
+==== Removing a Template
+
+
+
+
+
+
+== Monitoring of DataFlow
+
+=== Anatomy of a Processor
+
+=== Anatomy of a Process Group
+
+=== Anatomy of a Remote Process Group
+
+=== Status Page
+
+=== Historical Statics of a Component
+
+
+
+
+
+
+
+== Data Provenance
+
+=== Searching for Events
+
+=== Details of an Event
+
+=== Viewing FlowFile Content
+
+=== Replaying a FlowFile
+
+=== Viewing FlowFile Lineage
+==== Find Parents
+==== Expanding an Event
+
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a23c7f2e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3dc1bdd..9cccbf4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,8 +12,7 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+--><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
@@ -69,7 +68,8 @@
<module>nifi-mock</module>
<module>nar-bundles</module>
<module>assembly</module>
- </modules>
+ <module>nifi-docs</module>
+ </modules>
<scm>
<connection>scm:git:git://git.apache.org/incubator-nifi.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-nifi.git</developerConnection>
@@ -958,4 +958,4 @@
</plugin>
</plugins>
</build>
-</project>
+</project>
\ No newline at end of file
[08/31] incubator-nifi git commit: NIFI-150 updated replace text
plugin to avoid double addition of license header
Posted by mc...@apache.org.
NIFI-150 updated replace text plugin to avoid double addition of license header
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/782b2c18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/782b2c18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/782b2c18
Branch: refs/heads/NIFI-182
Commit: 782b2c18820b34a7f9ed1bb8df41dacaa928425e
Parents: d8911e7
Author: joewitt <jo...@apache.org>
Authored: Fri Dec 19 13:22:39 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Fri Dec 19 13:22:39 2014 -0500
----------------------------------------------------------------------
nifi-docs/pom.xml | 121 +++++++++++++++++++++++++------------------------
1 file changed, 62 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/782b2c18/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
index a8a546a..79ae0ac 100644
--- a/nifi-docs/pom.xml
+++ b/nifi-docs/pom.xml
@@ -1,60 +1,63 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
- <artifactId>nifi-docs</artifactId>
+ <artifactId>nifi-docs</artifactId>
- <name>nifi-docs</name>
+ <name>nifi-docs</name>
- <build>
- <plugins>
- <plugin>
- <groupId>org.asciidoctor</groupId>
- <artifactId>asciidoctor-maven-plugin</artifactId>
- <version>1.5.0</version>
- <executions>
- <execution>
- <id>output-html</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>process-asciidoc</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <backend>html</backend>
- <attributes>
- <skipFrontMatter>true</skipFrontMatter>
- </attributes>
- </configuration>
- </plugin>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.asciidoctor</groupId>
+ <artifactId>asciidoctor-maven-plugin</artifactId>
+ <version>1.5.0</version>
+ <executions>
+ <execution>
+ <id>output-html</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>process-asciidoc</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <backend>html</backend>
+ <attributes>
+ <skipFrontMatter>true</skipFrontMatter>
+ </attributes>
+ </configuration>
+ </plugin>
- <!-- This plugin is used to insert the Apache License into the output HMTL because
- AsciiDoc doesn't appear to provide a mechanism for doing this. -->
- <plugin>
- <groupId>com.google.code.maven-replacer-plugin</groupId>
- <artifactId>replacer</artifactId>
- <version>1.5.3</version>
- <executions>
- <execution>
- <phase>prepare-package</phase>
- <goals>
- <goal>replace</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <file>target/generated-docs/nifi-user-guide.html</file>
- <regexFlags>
- <regexFlag>DOTALL</regexFlag>
- </regexFlags>
- <token>(.*)</token>
- <value><!--
+ <!-- This plugin is used to insert the Apache License into the output HMTL because
+ AsciiDoc doesn't appear to provide a mechanism for doing this. -->
+ <plugin>
+ <groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <version>1.5.3</version>
+ <executions>
+ <execution>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>replace</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <file>target/generated-docs/nifi-user-guide.html</file>
+ <regex>true</regex>
+ <regexFlags>
+ <regexFlag>DOTALL</regexFlag>
+ <regexFlag>MULTILINE</regexFlag>
+ </regexFlags>
+ <token>^(.*)$</token>
+ <value>
+<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
@@ -62,7 +65,7 @@ The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -71,12 +74,12 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
$1
- </value>
- </configuration>
- </plugin>
- </plugins>
- </build>
+ </value>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
- <dependencies>
- </dependencies>
+ <dependencies>
+ </dependencies>
</project>
[05/31] incubator-nifi git commit: NIFI-185: Improved error message
Posted by mc...@apache.org.
NIFI-185: Improved error message
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/cbb17465
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/cbb17465
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/cbb17465
Branch: refs/heads/NIFI-182
Commit: cbb174658790b47faf68ed56343d0d001acea59e
Parents: 713312d
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 10:02:53 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 10:02:53 2014 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/controller/StandardProcessorNode.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/cbb17465/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
index 451dcfe..071be4d 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
@@ -1024,7 +1024,7 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
for (final Relationship relationship : getUndefinedRelationships()) {
if (!isAutoTerminated(relationship)) {
final ValidationResult error = new ValidationResult.Builder()
- .explanation(relationship.getName() + " is not defined")
+ .explanation("Relationship '" + relationship.getName() + "' is not connected to any component and is not auto-terminated")
.subject("Relationship " + relationship.getName())
.valid(false)
.build();
[11/31] incubator-nifi git commit: NIFI-150: Addressed issues not
pulling in the user guide appropriately
Posted by mc...@apache.org.
NIFI-150: Addressed issues not pulling in the user guide appropriately
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/d57b34c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/d57b34c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/d57b34c4
Branch: refs/heads/NIFI-182
Commit: d57b34c452c28c9523850539a3edc5787d76414d
Parents: 9c38f35
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 14:13:34 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 14:13:34 2014 -0500
----------------------------------------------------------------------
.../nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp | 3 ++-
.../web/nifi-web-docs/src/main/webapp/js/application.js | 5 +++++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d57b34c4/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp b/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
index f82cc6d..a899603 100644
--- a/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
+++ b/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
@@ -21,7 +21,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>NiFi Documentation</title>
- <script type="text/javascript" src="../nifi/js/jquery/jquery-1.7.min.js"></script>
+ <script type="text/javascript" src="../nifi/js/jquery/jquery-2.1.1.min.js"></script>
<script type="text/javascript" src="js/application.js"></script>
<link href="css/main.css" rel="stylesheet" type="text/css" />
<link href="css/component-usage.css" rel="stylesheet" type="text/css" />
@@ -41,6 +41,7 @@
<div class="header">Documents</div>
<div class="component-links">
<ul>
+ <li class="component-item"><a class="component-link user-guide" href="user-guide/nifi-user-guide.html" target="component-usage">User Guide</a></li>
</ul>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d57b34c4/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js b/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
index 1a60de6..e9b0499 100644
--- a/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
+++ b/nar-bundles/framework-bundle/framework/web/nifi-web-docs/src/main/webapp/js/application.js
@@ -256,6 +256,11 @@ $(document).ready(function () {
}
});
+ // listen for on the rest api and user guide
+ $('a.rest-api a.user-guide').on('click', function() {
+ selectComponent($(this).text());
+ });
+
// get the initial selection
var initialComponentLink = $('a.component-link:first');
var initialSelection = $('#initial-selection').text();
[07/31] incubator-nifi git commit: NIFI-150: Added additional
documentation
Posted by mc...@apache.org.
NIFI-150: Added additional documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/d8911e75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/d8911e75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/d8911e75
Branch: refs/heads/NIFI-182
Commit: d8911e75e6c44857416b96de2c54cb18eaaa2d98
Parents: c213895
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 12:28:14 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 12:28:14 2014 -0500
----------------------------------------------------------------------
.../src/main/asciidoc/images/iconAlert.png | Bin 0 -> 1396 bytes
.../src/main/asciidoc/images/iconDisable.png | Bin 0 -> 764 bytes
nifi-docs/src/main/asciidoc/images/iconEdit.png | Bin 0 -> 493 bytes
.../src/main/asciidoc/images/iconEnable.png | Bin 0 -> 667 bytes
.../src/main/asciidoc/images/iconNotSecure.png | Bin 0 -> 137 bytes
nifi-docs/src/main/asciidoc/images/iconRun.png | Bin 0 -> 538 bytes
.../src/main/asciidoc/images/iconSecure.png | Bin 0 -> 133 bytes
nifi-docs/src/main/asciidoc/images/iconStop.png | Bin 0 -> 402 bytes
.../asciidoc/images/iconTransmissionActive.png | Bin 0 -> 1330 bytes
.../images/iconTransmissionInactive.png | Bin 0 -> 1248 bytes
.../images/remote-group-ports-dialog.png | Bin 0 -> 21662 bytes
.../images/remote-port-connection-status.png | Bin 0 -> 57332 bytes
.../src/main/asciidoc/nifi-user-guide.adoc | 153 +++++++++++++++++--
13 files changed, 142 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconAlert.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconAlert.png b/nifi-docs/src/main/asciidoc/images/iconAlert.png
new file mode 100644
index 0000000..5020c31
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconAlert.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconDisable.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconDisable.png b/nifi-docs/src/main/asciidoc/images/iconDisable.png
new file mode 100644
index 0000000..17a4ee4
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconDisable.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconEdit.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconEdit.png b/nifi-docs/src/main/asciidoc/images/iconEdit.png
new file mode 100644
index 0000000..e191e72
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconEdit.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconEnable.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconEnable.png b/nifi-docs/src/main/asciidoc/images/iconEnable.png
new file mode 100644
index 0000000..58e3436
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconEnable.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconNotSecure.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconNotSecure.png b/nifi-docs/src/main/asciidoc/images/iconNotSecure.png
new file mode 100644
index 0000000..ae4c013
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconNotSecure.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconRun.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconRun.png b/nifi-docs/src/main/asciidoc/images/iconRun.png
new file mode 100644
index 0000000..02715e0
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconRun.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconSecure.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconSecure.png b/nifi-docs/src/main/asciidoc/images/iconSecure.png
new file mode 100644
index 0000000..a47388f
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconSecure.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconStop.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconStop.png b/nifi-docs/src/main/asciidoc/images/iconStop.png
new file mode 100644
index 0000000..4f8f9c7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconStop.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png b/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png
new file mode 100644
index 0000000..269cff8
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png b/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png
new file mode 100644
index 0000000..0e1fd7b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png b/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png
new file mode 100644
index 0000000..7b13b00
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png b/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png
new file mode 100644
index 0000000..2e7cecf
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d8911e75/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
index 8f01c01..62cf8a8 100644
--- a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
@@ -17,7 +17,7 @@
NiFi User Guide (Draft - Preview Version)
=========================================
Apache_NiFi_Team
-:toc: left
+:toc:
:icons:
@@ -351,7 +351,7 @@ resources that then are not usable by other Processors. This essentially provide
how much of the system's resources should be allocated to this Processor instead of other Processors. This field is available for
most Processors. There are, however, some types of Processors that can only be scheduled with a single Concurrent task.
-The `Run schedule` dictates how often this Processor should be scheduled to run. The valid values for this field depend on the selected
+The ``Run schedule'' dictates how often this Processor should be scheduled to run. The valid values for this field depend on the selected
Scheduling Strategy (see above). If using the Event driven Scheduling Strategy, this field is not available. When using the Timer driven
Scheduling Strategy, this value is a time duration specified by a number followed by a time unit. For example, `1 second` or `5 mins`.
The default value of `0 sec` means that the Processor should run as often as possible as long as it has data to process. This is true
@@ -480,49 +480,164 @@ prioritizers' list to the `Available prioritizers' list.
=== Processor Validation
Before trying to start a Processor, it's important to make sure that the Processor's configuration is valid.
+A status indicator is shown in the top-left of the Processor. If the Processor is invalid, the indicator
+will show a yellow Warning indicator with an exclamation mark indicating that there is a problem:
+image::invalid-processor.png["Invalid Processor"]
+
+In this case, hovering over the indicator icon with the mouse will provide a tooltip showing all of the validation
+failures for the Processor. Once all of the validation errors have been addressed, the status indicator will change
+to a Stop icon, indicating that the Processor is valid and ready to be start but currently is not running:
+
+image::valid-processor.png["Valid Processor"]
== Command and Control of DataFlow
+When a component is added to the NiFi canvas, it is in the Stopped state. In order to cause the component to
+be triggered, the component must be started. Once started, the component can be stopped at any time. From a
+Stopped state, the component can then be configured, started, or disabled.
+
=== Starting a Component
+In order to start a component, the following conditions must be met:
+- The component's configuration must be valid.
+- All defined Relationships for component must be connected to another component or auto-terminated.
+- The component must be stopped.
+- The component must be enabled.
+- The component must have no active tasks. For more information about active tasks, see the ``Anatomy of ...''
+sections under <<monitoring>> (<<processor_anatomy>>, <<process_group_anatomy>>, <<remote_group_anatomy>>).
+
+Components can be started by selecting all of the components to start and then clicking the Start icon (
+image:iconRun.png["Start"]
+) in the
+Actions Toolbar or by right-clicking a single component and choosing Start from the context menu.
+
+If starting a Process Group, all components within that Process Group (including child Process Groups) will
+be started, with the exception of those components that are invalid or disabled.
+
+Once started, the status indicator of a Processor will change to a Play symbol (
+image:iconRun.png["Run"]
+).
+
+
=== Stopping a Component
-=== Enabling a Component
+A component can be stopped any time that it is running. A component is stopped by right-clicking on the component
+and clicking Stop from the context menu, or by clicking the Stop icon (
+image:iconStop.png["Stop"]
+) in the Actions Toolbar.
-=== Disabling a Component
+If a Process Group is stopped, all of the components within the Process Group (including child Process Groups)
+will be stopped.
+Once stopped, the status indicator of a Processor will change to the Stop symbol (
+image:iconStop.png["Stop"]
+).
+Stopping a component does not interrupt its currently running tasks. Rather, it stops scheduling new tasks to
+be performed. The number of active tasks is shown in the top-right corner of the Processor (see <<processor_anatomy>>
+for more information).
+=== Enabling/Disabling a Component
+When a component is enabled, it is able to be started. Components may be disabled when part of a
+dataflow is still being assembled, for example, and as a result should not be started. Typically,
+if a component is not intended to be run, the component is disabled, rather than being left in the
+Stopped state. This helps to distinguish between components that are intentionally not running and
+those components that may have been stopped temporarily (for instance, to change the component's
+configuration) and inadvertently were never restarted.
+When it is desirable to re-enable a component, it can be enabled by selecting the component and
+clicking the Enable icon (
+image:iconEnable.png["Enable"]
+) in the Actions Toolbar. This is available only when the selected component or components are disabled.
+Alternatively, a component can be enabled by checking the checkbox next to the ``Enabled'' option in
+the Settings tab of the Processor configuration dialog or the configuration dialog for a Port.
+Once enabled, the component's status indicator will change to either Invalid (
+image:iconAlert.png["Invalid"]
+) or Stopped (
+image:iconStop.png["Stopped"]
+), depending on whether or not the component is valid.
-== Templates
+A component is then disabled by selecting the component and clicking the Disable icon (
+image:iconDisable.png["Disable"]
+) in the Actions Toolbar, or by clearing the checkbox next to the ``Enabled'' option in the Settings tab
+of the Processor configuration dialog or the configuration dialog for a Port.
-=== Creating a Template
+Only Ports and Processors can be enabled and disabled.
-=== Instantiating a Template
-=== Managing Templates
-==== Importing a Template
-==== Exporting a Template
-==== Removing a Template
+=== Remote Process Group Transmission
+
+Remote Process Groups provide a mechanism for sending data to or retrieving data from a remote instance
+of NiFi. When a Remote Process Group (RPG) is added to the canvas, it is added with the Transmision Disabled,
+as indicated by the icon (
+image:iconTransmissionInactive.png["Transmission Disabled"]
+) in the top-left corner. When Transmission is Disabled, it can be enabled by right-clicking on the
+RPG and clicking the ``Enable Transmission'' menu item. This will cause all ports for which there is a Connection
+to begin transmitting data. This will cause the status indicator to then change to the Transmission Enabled icon (
+image:iconTransmissionActive.png["Transmission Enabled"]
+).
+
+If there are problems communicating with the Remote Process Group, a Warning indicator (
+image:iconAlert.png["Warning"]
+) may instead be present in the top-left corner. Hovering over this Warning indicator with the mouse will provide
+more information about the problem.
+
+
+==== Individual Port Transmission
+
+There are times when the DFM may want to either enable or disable transmission for only a specific
+Port within the Remote Process Group. This can be accomplished by right-clicking on the Remote Process Group
+and choosing the ``Remote ports'' menu item. This provides a configuration dialog from which each Port can be
+configured:
+
+image::remote-group-ports-dialog.png["Remote Process Groups"]
+The left-hand side lists all of the Input Ports that the remote instance of NiFi allows data to be sent to.
+The right-hand side lists all of the Output Ports from which this instance is able to pull data.
+If the remote instance is using secure communications (the URL of the NiFi instance begins with `https://`,
+rather than `http://`), any Ports that the remote instance has not made available to this instance will not
+be shown.
+*Note*: If a Port that is expected to be shown is not shown in this dialog, ensure that the instance has proper
+permissions and that the Remote Process Group's flow is current. This can be checked by closing the Port
+Configuration Dialog and looking at the bottom-right corner of the Remote Process Group. The data at which
+the flow was last refresh is shown. If the flow appears to be outdated, it can be updated by right-clicking
+on the Remote Process Group and selecting ``Refresh flow.'' (See <<remote_group_anatomy>> for more information).
+Each Port is shown with the Port name, followed by its description, currently configured number of Concurrent
+tasks, and whether or not data sent to this port will be compressed. To the left of this information is a switch
+to turn the Port on or off. Those Ports that have no Connections attached to them are grayed out:
+image::remote-port-connection-status.png["Remote Port Statuses"]
+The on/off switch provides a mechanism to enable and disable transmission for each Port in the Remote
+Process Group independently. Those Ports that are connected but are not currently transmitting can be
+configured by clicking the pencil icon (
+image:iconEdit.png["Edit"]
+) below the on/off switch. Clicking this icon will allow the DFM to change the number of Concurrent tasks and whether
+or not compression should be used when transmitting data to or from this Port.
+
+
+
+
+
+[[monitoring]]
== Monitoring of DataFlow
+[[processor_anatomy]]
=== Anatomy of a Processor
+[[process_group_anatomy]]
=== Anatomy of a Process Group
+[[remote_group_anatomy]]
=== Anatomy of a Remote Process Group
=== Status Page
@@ -535,6 +650,22 @@ Before trying to start a Processor, it's important to make sure that the Process
+
+== Templates
+
+=== Creating a Template
+
+=== Instantiating a Template
+
+=== Managing Templates
+==== Importing a Template
+==== Exporting a Template
+==== Removing a Template
+
+
+
+
+
== Data Provenance
=== Searching for Events
[10/31] incubator-nifi git commit: NIFI-150: Removed old formatted
document and updated asciidoc version of user guide
Posted by mc...@apache.org.
NIFI-150: Removed old formatted document and updated asciidoc version of user guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/9c38f358
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/9c38f358
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/9c38f358
Branch: refs/heads/NIFI-182
Commit: 9c38f358b8a599648f7da13a6d2cc0f3ff879863
Parents: 62f3354
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 14:11:43 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 14:11:43 2014 -0500
----------------------------------------------------------------------
.../src/main/resources/docs/NiFi User Guide.odt | Bin 372498 -> 0 bytes
nifi-docs/src/main/asciidoc/nifi-user-guide.adoc | 5 +++++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/9c38f358/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt b/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt
deleted file mode 100644
index 08d0f11..0000000
Binary files a/nar-bundles/framework-bundle/framework/resources/src/main/resources/docs/NiFi User Guide.odt and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/9c38f358/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
index 20c5270..65878df 100644
--- a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
@@ -504,10 +504,15 @@ Stopped state, the component can then be configured, started, or disabled.
=== Starting a Component
In order to start a component, the following conditions must be met:
+
- The component's configuration must be valid.
+
- All defined Relationships for component must be connected to another component or auto-terminated.
+
- The component must be stopped.
+
- The component must be enabled.
+
- The component must have no active tasks. For more information about active tasks, see the ``Anatomy of ...''
sections under <<monitoring>> (<<processor_anatomy>>, <<process_group_anatomy>>, <<remote_group_anatomy>>).
[30/31] incubator-nifi git commit: NIFI-183 Updated LICENSE and
NOTICE based on thorough analysis of all dependencies. Removed HORNETQ and
c3p0 as they are incompatible with ASF licensing policies.
Posted by mc...@apache.org.
NIFI-183 Updated LICENSE and NOTICE based on thorough analysis of all dependencies. Removed HORNETQ and c3p0 as they are incompatible with ASF licensing policies.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/0d8da145
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/0d8da145
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/0d8da145
Branch: refs/heads/NIFI-182
Commit: 0d8da145bd5e4c152cb9bbefbb1fefaa1a96e790
Parents: 42f9cfb
Author: joewitt <jo...@apache.org>
Authored: Sun Dec 21 15:14:53 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sun Dec 21 15:14:53 2014 -0500
----------------------------------------------------------------------
LICENSE | 384 +++++++++++++++++++
NOTICE | 7 +
.../standard-bundle/jms-processors/pom.xml | 4 -
.../nifi/processors/jms/util/JmsFactory.java | 28 --
.../nifi/processors/jms/util/JmsProperties.java | 3 +-
.../index.html | 1 -
.../index.html | 1 -
.../index.html | 1 -
pom.xml | 10 +
9 files changed, 402 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index d645695..3757939 100644
--- a/LICENSE
+++ b/LICENSE
@@ -200,3 +200,387 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+The binary distribution of this product bundles 'Antlr 3' which is available
+under a "3-clause BSD" license. For details see http://www.antlr3.org/license.html
+
+ Copyright (c) 2010 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+ Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ Neither the name of the author nor the names of its contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'Paranamer Core' which is
+available under a "3-clause BSD" license.
+For details see http://paranamer.codehaus.org/paranamer
+
+ Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'xmlenc Library' which is
+available under The BSD 2-Clause license found here:
+http://www.opensource.org/licenses/bsd-license.php
+For details http://xmlenc.sourceforge.net
+
+The binary distribution of this product bundles 'Protocol Buffer Java API'
+which is available under The BSD 2-Clause license found here:
+http://www.opensource.org/licenses/bsd-license.php
+For details http://code.google.com/p/protobuf
+
+The binary distribution of this product bundles 'JZlib' which is
+available under a "3-clause BSD" license.
+For details see http://www.jcraft.com/jzlib/
+
+ Copyright (c) 2000-2011 ymnk, JCraft,Inc. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the distribution.
+
+ 3. The names of the authors may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+ INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+ OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'JSch' which is
+available under a "3-clause BSD" license.
+For details see http://www.jcraft.com/jsch/
+
+ Copyright (c) 2002-2014 Atsuhiko Yamanaka, JCraft,Inc.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the distribution.
+
+ 3. The names of the authors may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+ INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+ OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This product bundles 'Javascript D3 Library' which is available under a
+"3-clause BSD" license. For details see http://d3js.org/
+
+ Copyright (c) 2010-2014, Michael Bostock
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * The name Michael Bostock may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'BouncyCastle Provider'
+which is available under an MIT style license.
+For details see http://www.bouncycastle.org/java.html
+
+ Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org)
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation the
+ rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ sell copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ IN THE SOFTWARE.
+
+The binary distribution of this product bundles 'SLF4J' which is available
+under an MIT style license.
+For details see http://www.qos.ch
+
+ Copyright (c) 2004-2013 QOS.ch
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'jQuery Grid' which is available under the MIT License.
+http://www.opensource.org/licenses/mit-license.php
+For details see http://jqgrid.com/
+ Copyright (c) 2008, Tony Tomov, tony@trirand.com
+
+This product bundles 'CodeMirror' which is available under an MIT style license.
+For details see http://codemirror.net/doc/compress.html
+
+ Copyright (C) 2014 by Marijn Haverbeke <ma...@gmail.com> and others
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'jQuery UI' which is available under an MIT style license.
+For details see http://jqueryui.com
+
+ Copyright 2014 jQuery Foundation and other contributors,
+ http://jqueryui.com/
+
+ This software consists of voluntary contributions made by many
+ individuals (AUTHORS.txt, http://jqueryui.com/about) For exact
+ contribution history, see the revision history and logs, available
+ at http://jquery-ui.googlecode.com/svn/
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'SLICKGRID' which is available under an MIT style license.
+For details see http://github.com/mleibman/slickgrid
+ Copyright (c) 2010 Michael Leibman, http://github.com/mleibman/slickgrid
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'qTip2' which is available under an MIT style license.
+For details see http://qtip2.com
+
+ Copyright (c) 2012 Craig Michael Thompson
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'jQuery MiniColors' which is available under the MIT License.
+http://opensource.org/licenses/MIT
+For details see http://www.abeautifulsite.net/
+ Copyright Cory LaViska for A Beautiful Site, LLC. (http://www.abeautifulsite.net/)
+
+The binary distribution of this product bundles the following libraries:
+ Expression Languagee 3.0 API, JavaServer Pages(TM) API,
+ JavaServer Pages(TM) Standard Tag Library API, Expression Language 3.0,
+ JSP implementation, JavaServer Pages (TM) TagLib Implementation,
+ Java Servlet API
+which are all available under the CDDL 1.0 License.
+https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
+For details see https://glassfish.java.net/
+ Copyright (c) 2010-2014 Oracle and/or its affiliates. All rights reserved.
+
+The binary distribution of this product bundles the following libraries:
+ jersey-core, jersey-json, jersey-server, jersey-servlet, jersey-multipart,
+ jersey-spring, JAXB RI, JAXB API bundle for GlassFish V3,
+ MIME streaming extension, JavaMail API (compat)
+which are all available under the CDDL 1.1 License.
+http://glassfish.java.net/public/CDDL+GPL_1_1.html
+For details see https://glassfish.java.net/
+ Copyright (c) 2010-2014 Oracle and/or its affiliates. All rights reserved.
+
+The binary distribution of this product bundles 'H2 Database' which is available
+under the MPL 2.0 license.
+http://www.h2database.com/html/license.html#mpl2
+For details see http://www.h2database.com
+ This software contains unmodified binary redistributions for H2 database
+ engine (http://www.h2database.com/), which is dual licensed and available
+ under the MPL 2.0 (Mozilla Public License) or under the
+ EPL 1.0 (Eclipse Public License).
+ An original copy of the license agreement can be found
+ at: http://www.h2database.com/html/license.html
+
+The binary distribution of this product bundles 'Saxon-HE' which is available
+under the MPL 2.0 license http://www.mozilla.org/MPL/2.0/
+For details see http://saxonica.com/download/opensource.xml
+
+The binary distribution of this product bundles 'AspectJ Weaver' which is
+available under the EPL 1.0 license http://www.eclipse.org/legal/epl-v10.html
+For details see http://www.aspectj.org
+
+The binary distribution of this product bundles 'Logback' which is available
+under the EPL 1.0 license http://www.eclipse.org/legal/epl-v10.html
+For details see http://logback.qos.ch
+ Copyright (C) 1999-2012, QOS.ch. All rights reserved.
+
+The binary distribution of this product bundles 'XZ for Java' which is available
+in the 'public domain'. For details see http://tukaani.org/xz/java.html
+
+The binary distribution of this product bundles 'AOP Alliance' which is
+available in the 'public domain'.
+For details see http://aopalliance.sourceforge.net
+
+This product bundles 'json2.js' which is available in the 'public domain'.
+For details see https://github.com/douglascrockford/JSON-js
+
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index e02ae44..41ecce9 100644
--- a/NOTICE
+++ b/NOTICE
@@ -4,4 +4,11 @@ Copyright 2014 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+Please note that this product bundles software libraries which are covered
+by the following "weak copyleft" style licenses including;
+CDDL v1.0, CDDL v1.1, MPL v2.0, and EPL v1.0
+
+Please note that this product bundles software libraries which are licensed
+to the 'public domain'.
+
Please see LICENSE for additional copyright and licensing information.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/nar-bundles/standard-bundle/jms-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/pom.xml b/nar-bundles/standard-bundle/jms-processors/pom.xml
index 7cc7b17..6e8e379 100644
--- a/nar-bundles/standard-bundle/jms-processors/pom.xml
+++ b/nar-bundles/standard-bundle/jms-processors/pom.xml
@@ -47,10 +47,6 @@
<artifactId>activemq-client</artifactId>
</dependency>
<dependency>
- <groupId>org.hornetq</groupId>
- <artifactId>hornetq-jms-client</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
index 9a8da62..f425f3a 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
@@ -25,7 +25,6 @@ import static org.apache.nifi.processors.jms.util.JmsProperties.DESTINATION_TYPE
import static org.apache.nifi.processors.jms.util.JmsProperties.DESTINATION_TYPE_QUEUE;
import static org.apache.nifi.processors.jms.util.JmsProperties.DESTINATION_TYPE_TOPIC;
import static org.apache.nifi.processors.jms.util.JmsProperties.DURABLE_SUBSCRIPTION;
-import static org.apache.nifi.processors.jms.util.JmsProperties.HORNETQ_PROVIDER;
import static org.apache.nifi.processors.jms.util.JmsProperties.JMS_PROVIDER;
import static org.apache.nifi.processors.jms.util.JmsProperties.MESSAGE_SELECTOR;
import static org.apache.nifi.processors.jms.util.JmsProperties.PASSWORD;
@@ -35,8 +34,6 @@ import static org.apache.nifi.processors.jms.util.JmsProperties.USERNAME;
import java.io.IOException;
import java.io.ObjectOutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
@@ -66,11 +63,6 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.command.ActiveMQQueue;
import org.apache.activemq.command.ActiveMQTopic;
-import org.hornetq.api.core.TransportConfiguration;
-import org.hornetq.core.remoting.impl.netty.NettyConnectorFactory;
-import org.hornetq.jms.client.HornetQJMSConnectionFactory;
-import org.hornetq.jms.client.HornetQQueue;
-import org.hornetq.jms.client.HornetQTopic;
public class JmsFactory {
@@ -341,8 +333,6 @@ public class JmsFactory {
public static Queue createQueue(final String jmsProvider, final String queueName) {
switch (jmsProvider) {
- case HORNETQ_PROVIDER:
- return new HornetQQueue(queueName);
case ACTIVEMQ_PROVIDER:
default:
return new ActiveMQQueue(queueName);
@@ -352,8 +342,6 @@ public class JmsFactory {
private static Topic createTopic(final ProcessContext context) {
final String topicName = context.getProperty(DESTINATION_NAME).getValue();
switch (context.getProperty(JMS_PROVIDER).getValue()) {
- case HORNETQ_PROVIDER:
- return new HornetQTopic(topicName);
case ACTIVEMQ_PROVIDER:
default:
return new ActiveMQTopic(topicName);
@@ -374,22 +362,6 @@ public class JmsFactory {
factory.setSendTimeout(timeoutMillis);
return factory;
}
- case HORNETQ_PROVIDER: {
- final Map<String, Object> params = new HashMap<>();
-
- try {
- final URI uriObject = new URI(url);
- params.put("host", uriObject.getHost());
- params.put("port", uriObject.getPort());
-
- final TransportConfiguration transportConfig = new TransportConfiguration(NettyConnectorFactory.class.getName(), params);
- final HornetQJMSConnectionFactory factory = new HornetQJMSConnectionFactory(false, transportConfig);
- factory.setCallTimeout(timeoutMillis);
- return factory;
- } catch (final URISyntaxException e) {
- return null; // won't happen b/c we already validated the URL
- }
- }
default:
throw new IllegalArgumentException("Unknown JMS Provider: " + jmsProvider);
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsProperties.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsProperties.java b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsProperties.java
index 1a50c4e..ac19986 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsProperties.java
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsProperties.java
@@ -23,7 +23,6 @@ import org.apache.nifi.processor.util.StandardValidators;
public class JmsProperties {
public static final String ACTIVEMQ_PROVIDER = "ActiveMQ";
- public static final String HORNETQ_PROVIDER = "HornetQ";
public static final String ACK_MODE_CLIENT = "Client Acknowledge";
public static final String ACK_MODE_AUTO = "Auto Acknowledge";
@@ -41,7 +40,7 @@ public class JmsProperties {
.name("JMS Provider")
.description("The Provider used for the JMS Server")
.required(true)
- .allowableValues(ACTIVEMQ_PROVIDER, HORNETQ_PROVIDER)
+ .allowableValues(ACTIVEMQ_PROVIDER)
.defaultValue(ACTIVEMQ_PROVIDER)
.build();
public static final PropertyDescriptor URL = new PropertyDescriptor.Builder()
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSQueue/index.html
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSQueue/index.html b/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSQueue/index.html
index 58b674d..bcbf0dd 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSQueue/index.html
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSQueue/index.html
@@ -38,7 +38,6 @@
<li>This property specifies the provider used for the JMS server. Available options include:
<ul>
<li>ActiveMQ</li>
- <li>HornetQ</li>
</ul></li>
<li>Default value: ActiveMQ</li>
<li>Supports expression language: false</li>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSTopic/index.html
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSTopic/index.html b/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSTopic/index.html
index 876513d..3496df8 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSTopic/index.html
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.GetJMSTopic/index.html
@@ -38,7 +38,6 @@
<li>This property specifies the provider used for the JMS server. Available options include:
<ul>
<li>ActiveMQ</li>
- <li>HornetQ</li>
</ul></li>
<li>Supports expression language: false</li>
</ul></li>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.PutJMS/index.html
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.PutJMS/index.html b/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.PutJMS/index.html
index 52f2e1f..387d03c 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.PutJMS/index.html
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/resources/docs/org.apache.nifi.processors.jms.PutJMS/index.html
@@ -38,7 +38,6 @@
<li>This property specifies the provider used for the JMS server. Available options include:
<ul>
<li>ActiveMQ</li>
- <li>HornetQ</li>
</ul></li>
<li>Default value: ActiveMQ</li>
<li>Supports expression language: false</li>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0d8da145/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e0d9b47..9607386 100644
--- a/pom.xml
+++ b/pom.xml
@@ -248,6 +248,16 @@
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
<version>2.2.1</version>
+ <exclusions>
+ <!--
+ | Exclude the quartz 2.2.1 bundled version of c3p0 because it is lgpl licensed
+ | We also don't use the JDBC related features of quartz for which the dependency would matter
+ -->
+ <exclusion>
+ <groupId>c3p0</groupId>
+ <artifactId>c3p0</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey.contribs</groupId>
[16/31] incubator-nifi git commit: NIFI-150 updated to ensure docs
get built and end up in the assembly under docs.
Posted by mc...@apache.org.
NIFI-150 updated to ensure docs get built and end up in the assembly under docs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/21182ac2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/21182ac2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/21182ac2
Branch: refs/heads/NIFI-182
Commit: 21182ac260f6f40e65d53750804924cf5096370c
Parents: d850768
Author: joewitt <jo...@apache.org>
Authored: Fri Dec 19 16:34:25 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Fri Dec 19 16:34:25 2014 -0500
----------------------------------------------------------------------
assembly/pom.xml | 20 ++++++++++++++++
assembly/src/main/assembly/dependencies.xml | 7 ++++++
nifi-docs/pom.xml | 23 ++++++++++++++++---
nifi-docs/src/main/assembly/dependencies.xml | 28 +++++++++++++++++++++++
pom.xml | 8 +++++++
5 files changed, 83 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/21182ac2/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 17cf29b..887086c 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -43,6 +43,19 @@
<excludeTransitive>false</excludeTransitive>
</configuration>
</execution>
+ <execution>
+ <id>unpack-docs</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>generate-resources</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/generated-docs</outputDirectory>
+ <includeArtifactIds>nifi-docs</includeArtifactIds>
+ <includeGroupIds>org.apache.nifi</includeGroupIds>
+ <excludeTransitive>false</excludeTransitive>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
@@ -125,6 +138,13 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-docs</artifactId>
+ <classifier>resources</classifier>
+ <scope>provided</scope> <!-- Provided - we don't want the zip in the libs -->
+ <type>zip</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-nar</artifactId>
<type>nar</type>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/21182ac2/assembly/src/main/assembly/dependencies.xml
----------------------------------------------------------------------
diff --git a/assembly/src/main/assembly/dependencies.xml b/assembly/src/main/assembly/dependencies.xml
index f90127a..13e5b83 100644
--- a/assembly/src/main/assembly/dependencies.xml
+++ b/assembly/src/main/assembly/dependencies.xml
@@ -81,6 +81,13 @@
<fileMode>0640</fileMode>
<filtered>false</filtered>
</fileSet>
+ <fileSet>
+ <directory>${project.build.directory}/generated-docs/</directory>
+ <outputDirectory>docs</outputDirectory>
+ <directoryMode>0750</directoryMode>
+ <fileMode>0640</fileMode>
+ <filtered>false</filtered>
+ </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/21182ac2/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
index 979f1f8..1979b66 100644
--- a/nifi-docs/pom.xml
+++ b/nifi-docs/pom.xml
@@ -99,9 +99,26 @@ $1
</value>
</configuration>
</plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make shared resource</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/dependencies.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
-
- <dependencies>
- </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/21182ac2/nifi-docs/src/main/assembly/dependencies.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/assembly/dependencies.xml b/nifi-docs/src/main/assembly/dependencies.xml
new file mode 100644
index 0000000..8c41ead
--- /dev/null
+++ b/nifi-docs/src/main/assembly/dependencies.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>resources</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}/generated-docs</directory>
+ <outputDirectory>/user-guide/</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/21182ac2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9cccbf4..2ccdaa9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -710,6 +710,14 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-docs</artifactId>
+ <version>${project.version}</version>
+ <classifier>resources</classifier>
+ <scope>provided</scope> <!-- Provided - we don't want the zip in the libs -->
+ <type>zip</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-nar</artifactId>
<version>${project.version}</version>
<type>nar</type>
[25/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
NIFI-189 Collapsed several commons libs into one
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/1357a17b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/1357a17b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/1357a17b
Branch: refs/heads/NIFI-182
Commit: 1357a17b6752a78dd87c3de7b0323f22e52731f9
Parents: c0e4618
Author: joewitt <jo...@apache.org>
Authored: Sat Dec 20 00:40:14 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sat Dec 20 00:40:14 2014 -0500
----------------------------------------------------------------------
commons/core-flowfile-attributes/pom.xml | 29 -
.../flowfile/attributes/CoreAttributes.java | 72 ---
.../attributes/FlowFileAttributeKey.java | 21 -
commons/data-provenance-utils/pom.xml | 2 +-
commons/naive-search-ring-buffer/pom.xml | 30 -
.../apache/nifi/util/NaiveSearchRingBuffer.java | 135 ----
.../nifi/util/TestNaiveSearchRingBuffer.java | 72 ---
commons/nifi-file-utils/pom.xml | 35 --
.../java/org/apache/nifi/file/FileUtils.java | 612 ------------------
commons/nifi-stream-utils/.gitignore | 1 -
commons/nifi-stream-utils/pom.xml | 30 -
.../org/apache/nifi/io/BufferedInputStream.java | 37 --
.../apache/nifi/io/BufferedOutputStream.java | 140 -----
.../apache/nifi/io/ByteArrayInputStream.java | 250 --------
.../apache/nifi/io/ByteArrayOutputStream.java | 250 --------
.../apache/nifi/io/ByteCountingInputStream.java | 104 ----
.../nifi/io/ByteCountingOutputStream.java | 63 --
.../org/apache/nifi/io/DataOutputStream.java | 417 -------------
.../org/apache/nifi/io/GZIPOutputStream.java | 41 --
.../nifi/io/LeakyBucketStreamThrottler.java | 324 ----------
.../apache/nifi/io/NonCloseableInputStream.java | 56 --
.../nifi/io/NonCloseableOutputStream.java | 51 --
.../org/apache/nifi/io/NullOutputStream.java | 46 --
.../org/apache/nifi/io/StreamThrottler.java | 33 -
.../java/org/apache/nifi/io/StreamUtils.java | 257 --------
.../org/apache/nifi/io/ZipOutputStream.java | 38 --
.../exception/BytePatternNotFoundException.java | 28 -
.../io/util/NonThreadSafeCircularBuffer.java | 69 --
.../nifi/io/TestLeakyBucketThrottler.java | 144 -----
.../src/test/resources/logback-test.xml | 48 --
commons/nifi-utils/pom.xml | 4 +
.../flowfile/attributes/CoreAttributes.java | 72 +++
.../attributes/FlowFileAttributeKey.java | 21 +
.../apache/nifi/io/CompoundUpdateMonitor.java | 115 ----
.../org/apache/nifi/io/LastModifiedMonitor.java | 30 -
.../java/org/apache/nifi/io/MD5SumMonitor.java | 49 --
.../apache/nifi/io/SynchronousFileWatcher.java | 123 ----
.../java/org/apache/nifi/io/UpdateMonitor.java | 25 -
.../nifi/remote/StandardVersionNegotiator.java | 81 +++
.../apache/nifi/remote/VersionNegotiator.java | 65 ++
.../TransmissionDisabledException.java | 25 +
.../nifi/remote/io/CompressionInputStream.java | 184 ++++++
.../nifi/remote/io/CompressionOutputStream.java | 147 +++++
.../remote/io/InterruptableInputStream.java | 117 ++++
.../remote/io/InterruptableOutputStream.java | 81 +++
.../remote/io/socket/BufferStateManager.java | 111 ++++
.../io/socket/SocketChannelInputStream.java | 157 +++++
.../io/socket/SocketChannelOutputStream.java | 113 ++++
.../remote/io/socket/ssl/SSLSocketChannel.java | 602 ++++++++++++++++++
.../socket/ssl/SSLSocketChannelInputStream.java | 62 ++
.../ssl/SSLSocketChannelOutputStream.java | 53 ++
.../nifi/stream/io/BufferedInputStream.java | 37 ++
.../nifi/stream/io/BufferedOutputStream.java | 140 +++++
.../nifi/stream/io/ByteArrayInputStream.java | 250 ++++++++
.../nifi/stream/io/ByteArrayOutputStream.java | 250 ++++++++
.../nifi/stream/io/ByteCountingInputStream.java | 104 ++++
.../stream/io/ByteCountingOutputStream.java | 63 ++
.../apache/nifi/stream/io/DataOutputStream.java | 417 +++++++++++++
.../apache/nifi/stream/io/GZIPOutputStream.java | 41 ++
.../stream/io/LeakyBucketStreamThrottler.java | 324 ++++++++++
.../nifi/stream/io/NonCloseableInputStream.java | 56 ++
.../stream/io/NonCloseableOutputStream.java | 51 ++
.../apache/nifi/stream/io/NullOutputStream.java | 46 ++
.../apache/nifi/stream/io/StreamThrottler.java | 33 +
.../org/apache/nifi/stream/io/StreamUtils.java | 257 ++++++++
.../apache/nifi/stream/io/ZipOutputStream.java | 38 ++
.../exception/BytePatternNotFoundException.java | 28 +
.../io/util/NonThreadSafeCircularBuffer.java | 69 ++
.../apache/nifi/util/NaiveSearchRingBuffer.java | 135 ++++
.../org/apache/nifi/util/file/FileUtils.java | 623 +++++++++++++++++++
.../file/monitor/CompoundUpdateMonitor.java | 115 ++++
.../util/file/monitor/LastModifiedMonitor.java | 30 +
.../nifi/util/file/monitor/MD5SumMonitor.java | 51 ++
.../file/monitor/SynchronousFileWatcher.java | 123 ++++
.../nifi/util/file/monitor/UpdateMonitor.java | 25 +
.../org/apache/nifi/util/search/Search.java | 57 ++
.../org/apache/nifi/util/search/SearchTerm.java | 141 +++++
.../util/search/ahocorasick/AhoCorasick.java | 155 +++++
.../nifi/util/search/ahocorasick/Node.java | 72 +++
.../util/search/ahocorasick/SearchState.java | 63 ++
.../io/TestCompressionInputOutputStreams.java | 153 +++++
.../stream/io/TestLeakyBucketThrottler.java | 147 +++++
.../nifi/util/TestNaiveSearchRingBuffer.java | 72 +++
.../file/monitor/TestCompoundUpdateMonitor.java | 71 +++
.../monitor/TestSynchronousFileWatcher.java | 61 ++
.../timebuffer/TestCompoundUpdateMonitor.java | 75 ---
.../timebuffer/TestSynchronousFileWatcher.java | 64 --
.../src/test/resources/logback-test.xml | 32 +
commons/pom.xml | 6 -
commons/remote-communications-utils/pom.xml | 29 -
.../nifi/remote/StandardVersionNegotiator.java | 81 ---
.../apache/nifi/remote/VersionNegotiator.java | 65 --
.../TransmissionDisabledException.java | 25 -
.../nifi/remote/io/CompressionInputStream.java | 184 ------
.../nifi/remote/io/CompressionOutputStream.java | 147 -----
.../remote/io/InterruptableInputStream.java | 117 ----
.../remote/io/InterruptableOutputStream.java | 81 ---
.../remote/io/socket/BufferStateManager.java | 111 ----
.../io/socket/SocketChannelInputStream.java | 157 -----
.../io/socket/SocketChannelOutputStream.java | 113 ----
.../remote/io/socket/ssl/SSLSocketChannel.java | 602 ------------------
.../socket/ssl/SSLSocketChannelInputStream.java | 62 --
.../ssl/SSLSocketChannelOutputStream.java | 53 --
.../io/TestCompressionInputOutputStreams.java | 153 -----
commons/search-utils/pom.xml | 30 -
.../org/apache/nifi/util/search/Search.java | 57 --
.../org/apache/nifi/util/search/SearchTerm.java | 141 -----
.../util/search/ahocorasick/AhoCorasick.java | 155 -----
.../nifi/util/search/ahocorasick/Node.java | 72 ---
.../util/search/ahocorasick/SearchState.java | 63 --
commons/wali/pom.xml | 2 +-
.../org/wali/MinimalLockingWriteAheadLog.java | 4 +-
.../framework-bundle/framework/cluster/pom.xml | 2 +-
.../impl/FileBasedClusterNodeFirewall.java | 2 +-
.../nifi/cluster/flow/impl/DataFlowDaoImpl.java | 10 +-
.../impl/FileBasedClusterNodeFirewallTest.java | 3 +-
.../framework-bundle/framework/core-api/pom.xml | 6 +-
.../framework-bundle/framework/core/pom.xml | 8 -
.../nifi/controller/FileSystemSwapManager.java | 2 +-
.../apache/nifi/controller/FlowController.java | 2 +-
.../nifi/controller/FlowUnmarshaller.java | 2 +-
.../apache/nifi/controller/SnippetManager.java | 8 +-
.../nifi/controller/StandardFlowService.java | 2 +-
.../controller/StandardFlowSynchronizer.java | 2 +-
.../apache/nifi/controller/TemplateManager.java | 8 +-
.../repository/FileSystemRepository.java | 4 +-
.../repository/StandardProcessSession.java | 6 +-
.../repository/VolatileContentRepository.java | 4 +-
.../service/ControllerServiceLoader.java | 2 +-
.../StandardXMLFlowConfigurationDAO.java | 2 +-
.../nifi/remote/StandardRemoteProcessGroup.java | 2 +-
.../TestWriteAheadFlowFileRepository.java | 2 +-
.../repository/io/TestLimitedOutputStream.java | 3 +-
.../file-authorization-provider/pom.xml | 2 +-
.../FileAuthorizationProvider.java | 2 +-
.../FileAuthorizationProviderTest.java | 2 +-
.../framework/site-to-site/pom.xml | 12 -
.../remote/codec/StandardFlowFileCodec.java | 2 +-
.../remote/io/socket/SocketChannelInput.java | 4 +-
.../remote/io/socket/SocketChannelOutput.java | 4 +-
.../io/socket/ssl/SSLSocketChannelInput.java | 4 +-
.../io/socket/ssl/SSLSocketChannelOutput.java | 4 +-
.../apache/nifi/web/api/ProvenanceResource.java | 2 +-
.../nifi/web/util/ClientResponseUtils.java | 2 +-
.../hadoop-bundle/hdfs-processors/pom.xml | 6 +-
...lowFileStreamUnpackerSequenceFileWriter.java | 2 +-
.../apache/nifi/processors/hadoop/PutHDFS.java | 4 +-
.../hadoop/SequenceFileWriterImpl.java | 2 +-
.../hadoop/TarUnpackerSequenceFileWriter.java | 2 +-
.../hadoop/ZipUnpackerSequenceFileWriter.java | 2 +-
.../hadoop/util/OutputStreamWritable.java | 4 +-
.../monitor-threshold-bundle/processor/pom.xml | 8 -
.../processors/monitor/MonitorThreshold.java | 2 +-
nar-bundles/monitor-threshold-bundle/ui/pom.xml | 2 +-
.../persistent-provenance-repository/pom.xml | 8 -
.../nifi/provenance/StandardRecordReader.java | 4 +-
.../nifi/provenance/StandardRecordWriter.java | 6 +-
.../provenance/rollover/CompressionAction.java | 4 +-
.../provenance/serialization/RecordReaders.java | 2 +-
.../standard-bundle/jms-processors/pom.xml | 2 +-
.../apache/nifi/processors/jms/JmsConsumer.java | 2 +-
.../org/apache/nifi/processors/jms/PutJMS.java | 2 +-
.../nifi/processors/jms/util/JmsFactory.java | 2 +-
.../standard-bundle/standard-processors/pom.xml | 14 +-
.../processors/standard/CompressContent.java | 6 +-
.../processors/standard/EncryptContent.java | 2 +-
.../standard/EvaluateRegularExpression.java | 2 +-
.../nifi/processors/standard/EvaluateXPath.java | 4 +-
.../processors/standard/EvaluateXQuery.java | 4 +-
.../standard/ExecuteStreamCommand.java | 6 +-
.../nifi/processors/standard/HashContent.java | 4 +-
.../nifi/processors/standard/ListenHTTP.java | 4 +-
.../nifi/processors/standard/MergeContent.java | 8 +-
.../nifi/processors/standard/ModifyBytes.java | 2 +-
.../nifi/processors/standard/PostHTTP.java | 12 +-
.../processors/standard/PutFileTransfer.java | 2 +-
.../nifi/processors/standard/ReplaceText.java | 2 +-
.../standard/ReplaceTextWithMapping.java | 2 +-
.../processors/standard/RouteOnContent.java | 2 +-
.../nifi/processors/standard/ScanAttribute.java | 4 +-
.../nifi/processors/standard/ScanContent.java | 6 +-
.../nifi/processors/standard/SplitContent.java | 2 +-
.../nifi/processors/standard/SplitText.java | 8 +-
.../nifi/processors/standard/SplitXml.java | 2 +-
.../nifi/processors/standard/TransformXml.java | 2 +-
.../nifi/processors/standard/UnpackContent.java | 6 +-
.../standard/servlets/ListenHTTPServlet.java | 4 +-
.../standard/util/UDPStreamConsumer.java | 2 +-
.../processors/standard/TestMergeContent.java | 3 +-
.../processors/standard/TestScanContent.java | 4 +-
.../distributed-cache-client-service/pom.xml | 6 +-
.../DistributedMapCacheClientService.java | 4 +-
.../DistributedSetCacheClientService.java | 4 +-
.../cache/client/SSLCommsSession.java | 4 +-
.../cache/client/StandardCommsSession.java | 4 +-
.../distributed-cache-protocol/pom.xml | 2 +-
.../distributed-cache-server/pom.xml | 6 +-
.../cache/server/AbstractCacheServer.java | 4 +-
.../cache/server/SetCacheServer.java | 2 +-
.../cache/server/map/MapCacheServer.java | 2 +-
.../update-attribute-bundle/processor/pom.xml | 2 +-
nifi-mock/pom.xml | 4 -
pom.xml | 36 +-
203 files changed, 6371 insertions(+), 6667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/core-flowfile-attributes/pom.xml
----------------------------------------------------------------------
diff --git a/commons/core-flowfile-attributes/pom.xml b/commons/core-flowfile-attributes/pom.xml
deleted file mode 100644
index 5a82ffa..0000000
--- a/commons/core-flowfile-attributes/pom.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>nifi-core-flowfile-attributes</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>NiFi: Core FlowFile Attributes</name>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
----------------------------------------------------------------------
diff --git a/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
deleted file mode 100644
index 24f43ca..0000000
--- a/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.flowfile.attributes;
-
-public enum CoreAttributes implements FlowFileAttributeKey {
- /**
- * The flowfile's path indicates the relative directory to which a FlowFile belongs and does not
- * contain the filename
- */
- PATH("path"),
-
- /**
- * The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not
- * contain the filename
- */
- ABSOLUTE_PATH("absolute.path"),
-
- /**
- * The filename of the FlowFile. The filename should not contain any directory structure.
- */
- FILENAME("filename"),
-
- /**
- * A unique UUID assigned to this FlowFile
- */
- UUID("uuid"),
-
- /**
- * A numeric value indicating the FlowFile priority
- */
- PRIORITY("priority"),
-
- /**
- * The MIME Type of this FlowFile
- */
- MIME_TYPE("mime.type"),
-
- /**
- * Specifies the reason that a FlowFile is being discarded
- */
- DISCARD_REASON("discard.reason"),
-
- /**
- * Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile.
- */
- ALTERNATE_IDENTIFIER("alternate.identifier");
-
- private final String key;
- private CoreAttributes(final String key) {
- this.key = key;
- }
-
- @Override
- public String key() {
- return key;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
----------------------------------------------------------------------
diff --git a/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
deleted file mode 100644
index cc6c28e..0000000
--- a/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.flowfile.attributes;
-
-public interface FlowFileAttributeKey {
- String key();
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/data-provenance-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/pom.xml b/commons/data-provenance-utils/pom.xml
index 162d4f3..0024b70 100644
--- a/commons/data-provenance-utils/pom.xml
+++ b/commons/data-provenance-utils/pom.xml
@@ -34,7 +34,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/naive-search-ring-buffer/pom.xml
----------------------------------------------------------------------
diff --git a/commons/naive-search-ring-buffer/pom.xml b/commons/naive-search-ring-buffer/pom.xml
deleted file mode 100644
index c9628f0..0000000
--- a/commons/naive-search-ring-buffer/pom.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>naive-search-ring-buffer</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>NiFi Ring Buffer</name>
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/naive-search-ring-buffer/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
----------------------------------------------------------------------
diff --git a/commons/naive-search-ring-buffer/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java b/commons/naive-search-ring-buffer/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
deleted file mode 100644
index 85bfd96..0000000
--- a/commons/naive-search-ring-buffer/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util;
-
-import java.util.Arrays;
-
-/**
- * <p>
- * A RingBuffer that can be used to scan byte sequences for subsequences.
- * </p>
- *
- * <p>
- * This class implements an efficient naive search algorithm, which allows the
- * user of the library to identify byte sequences in a stream on-the-fly so that
- * the stream can be segmented without having to buffer the data.
- * </p>
- *
- * <p>
- * The intended usage paradigm is:
- * <code>
- * <pre>
- * final byte[] searchSequence = ...;
- * final CircularBuffer buffer = new CircularBuffer(searchSequence);
- * while ((int nextByte = in.read()) > 0) {
- * if ( buffer.addAndCompare(nextByte) ) {
- * // This byte is the last byte in the given sequence
- * } else {
- * // This byte does not complete the given sequence
- * }
- * }
- * </pre>
- * </code>
- * </p>
- */
-public class NaiveSearchRingBuffer {
-
- private final byte[] lookingFor;
- private final int[] buffer;
- private int insertionPointer = 0;
- private int bufferSize = 0;
-
- public NaiveSearchRingBuffer(final byte[] lookingFor) {
- this.lookingFor = lookingFor;
- this.buffer = new int[lookingFor.length];
- Arrays.fill(buffer, -1);
- }
-
- /**
- * Returns the contents of the internal buffer, which represents the last X
- * bytes added to the buffer, where X is the minimum of the number of bytes
- * added to the buffer or the length of the byte sequence for which we are
- * looking
- *
- * @return
- */
- public byte[] getBufferContents() {
- final int contentLength = Math.min(lookingFor.length, bufferSize);
- final byte[] contents = new byte[contentLength];
- for (int i = 0; i < contentLength; i++) {
- final byte nextByte = (byte) buffer[(insertionPointer + i) % lookingFor.length];
- contents[i] = nextByte;
- }
- return contents;
- }
-
- /**
- * Returns the oldest byte in the buffer
- *
- * @return
- */
- public int getOldestByte() {
- return buffer[insertionPointer];
- }
-
- /**
- * Returns <code>true</code> if the number of bytes that have been added to
- * the buffer is at least equal to the length of the byte sequence for which
- * we are searching
- *
- * @return
- */
- public boolean isFilled() {
- return bufferSize >= buffer.length;
- }
-
- /**
- * Clears the internal buffer so that a new search may begin
- */
- public void clear() {
- Arrays.fill(buffer, -1);
- insertionPointer = 0;
- bufferSize = 0;
- }
-
- /**
- * Add the given byte to the buffer and notify whether or not the byte
- * completes the desired byte sequence.
- *
- * @param data
- * @return <code>true</code> if this byte completes the byte sequence,
- * <code>false</code> otherwise.
- */
- public boolean addAndCompare(final byte data) {
- buffer[insertionPointer] = data;
- insertionPointer = (insertionPointer + 1) % lookingFor.length;
-
- bufferSize++;
- if (bufferSize < lookingFor.length) {
- return false;
- }
-
- for (int i = 0; i < lookingFor.length; i++) {
- final byte compare = (byte) buffer[(insertionPointer + i) % lookingFor.length];
- if (compare != lookingFor[i]) {
- return false;
- }
- }
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/naive-search-ring-buffer/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
----------------------------------------------------------------------
diff --git a/commons/naive-search-ring-buffer/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java b/commons/naive-search-ring-buffer/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
deleted file mode 100644
index 0838e96..0000000
--- a/commons/naive-search-ring-buffer/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import org.junit.Test;
-
-public class TestNaiveSearchRingBuffer {
-
- @Test
- public void testAddAndCompare() {
- final byte[] pattern = new byte[]{
- '\r', '0', 38, 48
- };
-
- final byte[] search = new byte[]{
- '\r', '0', 38, 58, 58, 83, 78, '\r', '0', 38, 48, 83, 92, 78, 4, 38
- };
-
- final NaiveSearchRingBuffer circ = new NaiveSearchRingBuffer(pattern);
- int counter = -1;
- for (final byte b : search) {
- counter++;
- final boolean matched = circ.addAndCompare(b);
- if (counter == 10) {
- assertTrue(matched);
- } else {
- assertFalse(matched);
- }
- }
- }
-
- @Test
- public void testGetOldestByte() {
- final byte[] pattern = new byte[]{
- '\r', '0', 38, 48
- };
-
- final byte[] search = new byte[]{
- '\r', '0', 38, 58, 58, 83, 78, (byte) 223, (byte) 227, (byte) 250, '\r', '0', 38, 48, 83, 92, 78, 4, 38
- };
-
- final NaiveSearchRingBuffer circ = new NaiveSearchRingBuffer(pattern);
- int counter = -1;
- for (final byte b : search) {
- counter++;
- final boolean matched = circ.addAndCompare(b);
- if (counter == 13) {
- assertTrue(matched);
- } else {
- assertFalse(matched);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-file-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/nifi-file-utils/pom.xml b/commons/nifi-file-utils/pom.xml
deleted file mode 100644
index 2af4d63..0000000
--- a/commons/nifi-file-utils/pom.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>nifi-file-utils</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>NiFi File Utils</name>
-
- <dependencies>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-file-utils/src/main/java/org/apache/nifi/file/FileUtils.java
----------------------------------------------------------------------
diff --git a/commons/nifi-file-utils/src/main/java/org/apache/nifi/file/FileUtils.java b/commons/nifi-file-utils/src/main/java/org/apache/nifi/file/FileUtils.java
deleted file mode 100644
index 8920493..0000000
--- a/commons/nifi-file-utils/src/main/java/org/apache/nifi/file/FileUtils.java
+++ /dev/null
@@ -1,612 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.file;
-
-import java.io.BufferedInputStream;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Random;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import org.slf4j.Logger;
-
-/**
- * A utility class containing a few useful static methods to do typical IO
- * operations.
- *
- * @author unattributed
- */
-public class FileUtils {
-
- public static final long TRANSFER_CHUNK_SIZE_BYTES = 1024 * 1024 * 8; //8 MB chunks
- public static final long MILLIS_BETWEEN_ATTEMPTS = 50L;
-
- /**
- * Closes the given closeable quietly - no logging, no exceptions...
- *
- * @param closeable
- */
- public static void closeQuietly(final Closeable closeable) {
- if (null != closeable) {
- try {
- closeable.close();
- } catch (final IOException io) {/*IGNORE*/
-
- }
- }
- }
-
- /**
- * Releases the given lock quietly - no logging, no exception
- *
- * @param lock
- */
- public static void releaseQuietly(final FileLock lock) {
- if (null != lock) {
- try {
- lock.release();
- } catch (final IOException io) {
- /*IGNORE*/
- }
- }
- }
-
- public static void ensureDirectoryExistAndCanAccess(final File dir) throws IOException {
- if (dir.exists() && !dir.isDirectory()) {
- throw new IOException(dir.getAbsolutePath() + " is not a directory");
- } else if (!dir.exists()) {
- final boolean made = dir.mkdirs();
- if (!made) {
- throw new IOException(dir.getAbsolutePath() + " could not be created");
- }
- }
- if (!(dir.canRead() && dir.canWrite())) {
- throw new IOException(dir.getAbsolutePath() + " directory does not have read/write privilege");
- }
- }
-
- /**
- * Deletes the given file. If the given file exists but could not be deleted
- * this will be printed as a warning to the given logger
- *
- * @param file
- * @param logger
- * @return
- */
- public static boolean deleteFile(final File file, final Logger logger) {
- return FileUtils.deleteFile(file, logger, 1);
- }
-
- /**
- * Deletes the given file. If the given file exists but could not be deleted
- * this will be printed as a warning to the given logger
- *
- * @param file
- * @param logger
- * @param attempts indicates how many times an attempt to delete should be
- * made
- * @return true if given file no longer exists
- */
- public static boolean deleteFile(final File file, final Logger logger, final int attempts) {
- if(file == null){
- return false;
- }
- boolean isGone = false;
- try {
- if (file.exists()) {
- final int effectiveAttempts = Math.max(1, attempts);
- for (int i = 0; i < effectiveAttempts && !isGone; i++) {
- isGone = file.delete() || !file.exists();
- if (!isGone && (effectiveAttempts - i) > 1) {
- FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS);
- }
- }
- if (!isGone && logger != null) {
- logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath());
- }
- }
- } catch (final Throwable t) {
- if (logger != null) {
- logger.warn("Unable to delete file: '" + file.getAbsolutePath() + "' due to " + t);
- }
- }
- return isGone;
- }
-
- /**
- * Deletes all of the given files. If any exist and cannot be deleted that
- * will be printed at warn to the given logger.
- *
- * @param files can be null
- * @param logger can be null
- */
- public static void deleteFile(final List<File> files, final Logger logger) {
- FileUtils.deleteFile(files, logger, 1);
- }
-
- /**
- * Deletes all of the given files. If any exist and cannot be deleted that
- * will be printed at warn to the given logger.
- *
- * @param files can be null
- * @param logger can be null
- * @param attempts indicates how many times an attempt should be made to
- * delete each file
- */
- public static void deleteFile(final List<File> files, final Logger logger, final int attempts) {
- if (null == files || files.isEmpty()) {
- return;
- }
- final int effectiveAttempts = Math.max(1, attempts);
- for (final File file : files) {
- try {
- boolean isGone = false;
- for (int i = 0; i < effectiveAttempts && !isGone; i++) {
- isGone = file.delete() || !file.exists();
- if (!isGone && (effectiveAttempts - i) > 1) {
- FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS);
- }
- }
- if (!isGone && logger != null) {
- logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath());
- }
- } catch (final Throwable t) {
- if (null != logger) {
- logger.warn("Unable to delete file given from path: '" + file.getPath() + "' due to " + t);
- }
- }
- }
- }
-
- /**
- * Deletes all files (not directories..) in the given directory (non
- * recursive) that match the given filename filter. If any file cannot be
- * deleted then this is printed at warn to the given logger.
- *
- * @param directory
- * @param filter if null then no filter is used
- * @param logger
- */
- public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger) {
- FileUtils.deleteFilesInDir(directory, filter, logger, false);
- }
-
- /**
- * Deletes all files (not directories) in the given directory (recursive)
- * that match the given filename filter. If any file cannot be deleted then
- * this is printed at warn to the given logger.
- *
- * @param directory
- * @param filter if null then no filter is used
- * @param logger
- * @param recurse
- */
- public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse) {
- FileUtils.deleteFilesInDir(directory, filter, logger, recurse, false);
- }
-
- /**
- * Deletes all files (not directories) in the given directory (recursive)
- * that match the given filename filter. If any file cannot be deleted then
- * this is printed at warn to the given logger.
- *
- * @param directory
- * @param filter if null then no filter is used
- * @param logger
- * @param recurse
- * @param deleteEmptyDirectories default is false; if true will delete
- * directories found that are empty
- */
- public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse, final boolean deleteEmptyDirectories) {
- // ensure the specified directory is actually a directory and that it exists
- if (null != directory && directory.isDirectory()) {
- final File ingestFiles[] = directory.listFiles();
- for (File ingestFile : ingestFiles) {
- boolean process = (filter == null) ? true : filter.accept(directory, ingestFile.getName());
- if (ingestFile.isFile() && process) {
- FileUtils.deleteFile(ingestFile, logger, 3);
- }
- if (ingestFile.isDirectory() && recurse) {
- FileUtils.deleteFilesInDir(ingestFile, filter, logger, recurse, deleteEmptyDirectories);
- if (deleteEmptyDirectories && ingestFile.list().length == 0) {
- FileUtils.deleteFile(ingestFile, logger, 3);
- }
- }
- }
- }
- }
-
- /**
- * Deletes given files.
- *
- * @param files
- * @param recurse will recurse
- * @throws IOException
- */
- public static void deleteFiles(final Collection<File> files, final boolean recurse) throws IOException {
- for (final File file : files) {
- FileUtils.deleteFile(file, recurse);
- }
- }
-
- public static void deleteFile(final File file, final boolean recurse) throws IOException {
- if (file.isDirectory() && recurse) {
- FileUtils.deleteFiles(Arrays.asList(file.listFiles()), recurse);
- }
- //now delete the file itself regardless of whether it is plain file or a directory
- if (!FileUtils.deleteFile(file, null, 5)) {
- throw new IOException("Unable to delete " + file.getAbsolutePath());
- }
- }
-
- /**
- * Randomly generates a sequence of bytes and overwrites the contents of the
- * file a number of times. The file is then deleted.
- *
- * @param file File to be overwritten a number of times and, ultimately,
- * deleted
- * @param passes Number of times file should be overwritten
- * @throws IOException if something makes shredding or deleting a problem
- */
- public static void shredFile(final File file, final int passes)
- throws IOException {
- final Random generator = new Random();
- final long fileLength = file.length();
- final int byteArraySize = (int) Math.min(fileLength, 1048576); // 1MB
- final byte[] b = new byte[byteArraySize];
- final long numOfRandomWrites = (fileLength / b.length) + 1;
- final FileOutputStream fos = new FileOutputStream(file);
- try {
- // Over write file contents (passes) times
- final FileChannel channel = fos.getChannel();
- for (int i = 0; i < passes; i++) {
- generator.nextBytes(b);
- for (int j = 0; j <= numOfRandomWrites; j++) {
- fos.write(b);
- }
- fos.flush();
- channel.position(0);
- }
- // Write out "0" for each byte in the file
- Arrays.fill(b, (byte) 0);
- for (int j = 0; j < numOfRandomWrites; j++) {
- fos.write(b);
- }
- fos.flush();
- fos.close();
- // Try to delete the file a few times
- if (!FileUtils.deleteFile(file, null, 5)) {
- throw new IOException("Failed to delete file after shredding");
- }
-
- } finally {
- FileUtils.closeQuietly(fos);
- }
- }
-
- public static long copy(final InputStream in, final OutputStream out) throws IOException {
- final byte[] buffer = new byte[65536];
- long copied = 0L;
- int len;
- while ((len = in.read(buffer)) > 0) {
- out.write(buffer, 0, len);
- copied += len;
- }
-
- return copied;
- }
-
- public static long copyBytes(final byte[] bytes, final File destination, final boolean lockOutputFile) throws FileNotFoundException, IOException {
- FileOutputStream fos = null;
- FileLock outLock = null;
- long fileSize = 0L;
- try {
- fos = new FileOutputStream(destination);
- final FileChannel out = fos.getChannel();
- if (lockOutputFile) {
- outLock = out.tryLock(0, Long.MAX_VALUE, false);
- if (null == outLock) {
- throw new IOException("Unable to obtain exclusive file lock for: " + destination.getAbsolutePath());
- }
- }
- fos.write(bytes);
- fos.flush();
- fileSize = bytes.length;
- } finally {
- FileUtils.releaseQuietly(outLock);
- FileUtils.closeQuietly(fos);
- }
- return fileSize;
- }
-
- /**
- * Copies the given source file to the given destination file. The given
- * destination will be overwritten if it already exists.
- *
- * @param source
- * @param destination
- * @param lockInputFile if true will lock input file during copy; if false
- * will not
- * @param lockOutputFile if true will lock output file during copy; if false
- * will not
- * @param move if true will perform what is effectively a move operation
- * rather than a pure copy. This allows for potentially highly efficient
- * movement of the file but if not possible this will revert to a copy then
- * delete behavior. If false, then the file is copied and the source file is
- * retained. If a true rename/move occurs then no lock is held during that
- * time.
- * @param logger if failures occur, they will be logged to this logger if
- * possible. If this logger is null, an IOException will instead be thrown,
- * indicating the problem.
- * @return long number of bytes copied
- * @throws FileNotFoundException if the source file could not be found
- * @throws IOException
- * @throws SecurityException if a security manager denies the needed file
- * operations
- */
- public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final boolean move, final Logger logger) throws FileNotFoundException, IOException {
-
- FileInputStream fis = null;
- FileOutputStream fos = null;
- FileLock inLock = null;
- FileLock outLock = null;
- long fileSize = 0L;
- if (!source.canRead()) {
- throw new IOException("Must at least have read permission");
-
- }
- if (move && source.renameTo(destination)) {
- fileSize = destination.length();
- } else {
- try {
- fis = new FileInputStream(source);
- fos = new FileOutputStream(destination);
- final FileChannel in = fis.getChannel();
- final FileChannel out = fos.getChannel();
- if (lockInputFile) {
- inLock = in.tryLock(0, Long.MAX_VALUE, true);
- if (null == inLock) {
- throw new IOException("Unable to obtain shared file lock for: " + source.getAbsolutePath());
- }
- }
- if (lockOutputFile) {
- outLock = out.tryLock(0, Long.MAX_VALUE, false);
- if (null == outLock) {
- throw new IOException("Unable to obtain exclusive file lock for: " + destination.getAbsolutePath());
- }
- }
- long bytesWritten = 0;
- do {
- bytesWritten += out.transferFrom(in, bytesWritten, TRANSFER_CHUNK_SIZE_BYTES);
- fileSize = in.size();
- } while (bytesWritten < fileSize);
- out.force(false);
- FileUtils.closeQuietly(fos);
- FileUtils.closeQuietly(fis);
- fos = null;
- fis = null;
- if (move && !FileUtils.deleteFile(source, null, 5)) {
- if (logger == null) {
- FileUtils.deleteFile(destination, null, 5);
- throw new IOException("Could not remove file " + source.getAbsolutePath());
- } else {
- logger.warn("Configured to delete source file when renaming/move not successful. However, unable to delete file at: " + source.getAbsolutePath());
- }
- }
- } finally {
- FileUtils.releaseQuietly(inLock);
- FileUtils.releaseQuietly(outLock);
- FileUtils.closeQuietly(fos);
- FileUtils.closeQuietly(fis);
- }
- }
- return fileSize;
- }
-
- /**
- * Copies the given source file to the given destination file. The given
- * destination will be overwritten if it already exists.
- *
- * @param source
- * @param destination
- * @param lockInputFile if true will lock input file during copy; if false
- * will not
- * @param lockOutputFile if true will lock output file during copy; if false
- * will not
- * @param logger
- * @return long number of bytes copied
- * @throws FileNotFoundException if the source file could not be found
- * @throws IOException
- * @throws SecurityException if a security manager denies the needed file
- * operations
- */
- public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final Logger logger) throws FileNotFoundException, IOException {
- return FileUtils.copyFile(source, destination, lockInputFile, lockOutputFile, false, logger);
- }
-
- public static long copyFile(final File source, final OutputStream stream, final boolean closeOutputStream, final boolean lockInputFile) throws FileNotFoundException, IOException {
- FileInputStream fis = null;
- FileLock inLock = null;
- long fileSize = 0L;
- try {
- fis = new FileInputStream(source);
- final FileChannel in = fis.getChannel();
- if (lockInputFile) {
- inLock = in.tryLock(0, Long.MAX_VALUE, true);
- if (inLock == null) {
- throw new IOException("Unable to obtain exclusive file lock for: " + source.getAbsolutePath());
- }
-
- }
-
- byte[] buffer = new byte[1 << 18]; //256 KB
- int bytesRead = -1;
- while ((bytesRead = fis.read(buffer)) != -1) {
- stream.write(buffer, 0, bytesRead);
- }
- in.force(false);
- stream.flush();
- fileSize = in.size();
- } finally {
- FileUtils.releaseQuietly(inLock);
- FileUtils.closeQuietly(fis);
- if (closeOutputStream) {
- FileUtils.closeQuietly(stream);
- }
- }
- return fileSize;
- }
-
- public static long copyFile(final InputStream stream, final File destination, final boolean closeInputStream, final boolean lockOutputFile) throws FileNotFoundException, IOException {
- final Path destPath = destination.toPath();
- final long size = Files.copy(stream, destPath);
- if (closeInputStream) {
- stream.close();
- }
- return size;
- }
-
- /**
- * Renames the given file from the source path to the destination path. This
- * handles multiple attempts. This should only be used to rename within a
- * given directory. Renaming across directories might not work well. See the
- * <code>File.renameTo</code> for more information.
- *
- * @param source the file to rename
- * @param destination the file path to rename to
- * @param maxAttempts the max number of attempts to attempt the rename
- * @throws IOException if rename isn't successful
- */
- public static void renameFile(final File source, final File destination, final int maxAttempts) throws IOException {
- FileUtils.renameFile(source, destination, maxAttempts, false);
- }
-
- /**
- * Renames the given file from the source path to the destination path. This
- * handles multiple attempts. This should only be used to rename within a
- * given directory. Renaming across directories might not work well. See the
- * <code>File.renameTo</code> for more information.
- *
- * @param source the file to rename
- * @param destination the file path to rename to
- * @param maxAttempts the max number of attempts to attempt the rename
- * @param replace if true and a rename attempt fails will check if a file is
- * already at the destination path. If so it will delete that file and
- * attempt the rename according the remaining maxAttempts. If false, any
- * conflicting files will be left as they were and the rename attempts will
- * fail if conflicting.
- * @throws IOException if rename isn't successful
- */
- public static void renameFile(final File source, final File destination, final int maxAttempts, final boolean replace) throws IOException {
- final int attempts = (replace || maxAttempts < 1) ? Math.max(2, maxAttempts) : maxAttempts;
- boolean renamed = false;
- for (int i = 0; i < attempts; i++) {
- renamed = source.renameTo(destination);
- if (!renamed) {
- FileUtils.deleteFile(destination, null, 5);
- } else {
- break; //rename has succeeded
- }
- }
- if (!renamed) {
- throw new IOException("Attempted " + maxAttempts + " times but unable to rename from \'" + source.getPath() + "\' to \'" + destination.getPath() + "\'");
-
- }
- }
-
- public static void sleepQuietly(final long millis) {
- try {
- Thread.sleep(millis);
- } catch (final InterruptedException ex) {
- /* do nothing */
- }
- }
-
- /**
- * Syncs a primary copy of a file with the copy in the restore directory. If
- * the restore directory does not have a file and the primary has a file,
- * the the primary's file is copied to the restore directory. Else if the
- * restore directory has a file, but the primary does not, then the
- * restore's file is copied to the primary directory. Else if the primary
- * file is different than the restore file, then an IllegalStateException is
- * thrown. Otherwise, if neither file exists, then no syncing is performed.
- *
- * @param primaryFile the primary file
- * @param restoreFile the restore file
- * @param logger a logger
- * @throws IOException if an I/O problem was encountered during syncing
- * @throws IllegalStateException if the primary and restore copies exist but
- * are different
- */
- public static void syncWithRestore(final File primaryFile, final File restoreFile, final Logger logger)
- throws IOException {
-
- if (primaryFile.exists() && !restoreFile.exists()) {
- // copy primary file to restore
- copyFile(primaryFile, restoreFile, false, false, logger);
- } else if (restoreFile.exists() && !primaryFile.exists()) {
- // copy restore file to primary
- copyFile(restoreFile, primaryFile, false, false, logger);
- } else if (primaryFile.exists() && restoreFile.exists() && !isSame(primaryFile, restoreFile)) {
- throw new IllegalStateException(String.format("Primary file '%s' is different than restore file '%s'",
- primaryFile.getAbsoluteFile(), restoreFile.getAbsolutePath()));
- }
- }
-
- /**
- * Returns true if the given files are the same according to their MD5 hash.
- *
- * @param file1 a file
- * @param file2 a file
- * @return true if the files are the same; false otherwise
- * @throws IOException if the MD5 hash could not be computed
- */
- public static boolean isSame(final File file1, final File file2) throws IOException {
- return Arrays.equals(computeMd5Digest(file1), computeMd5Digest(file2));
- }
-
- /**
- * Returns the MD5 hash of the given file.
- *
- * @param file a file
- * @return the MD5 hash
- * @throws IOException if the MD5 hash could not be computed
- */
- public static byte[] computeMd5Digest(final File file) throws IOException {
- BufferedInputStream bis = null;
- try {
- bis = new BufferedInputStream(new FileInputStream(file));
- return DigestUtils.md5(bis);
- } finally {
- FileUtils.closeQuietly(bis);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/.gitignore
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/.gitignore b/commons/nifi-stream-utils/.gitignore
deleted file mode 100755
index ea8c4bf..0000000
--- a/commons/nifi-stream-utils/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/target
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/pom.xml b/commons/nifi-stream-utils/pom.xml
deleted file mode 100644
index ae06436..0000000
--- a/commons/nifi-stream-utils/pom.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>nifi-stream-utils</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>NiFi Stream Utils</name>
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedInputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedInputStream.java
deleted file mode 100644
index 57adb8c..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedInputStream.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.InputStream;
-
-/**
- * This class is a slight modification of the BufferedInputStream in the java.io
- * package. The modification is that this implementation does not provide
- * synchronization on method calls, which means that this class is not suitable
- * for use by multiple threads. However, the absence of these synchronized
- * blocks results in potentially much better performance.
- */
-public class BufferedInputStream extends java.io.BufferedInputStream {
-
- public BufferedInputStream(final InputStream in) {
- super(in);
- }
-
- public BufferedInputStream(final InputStream in, final int size) {
- super(in, size);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedOutputStream.java
deleted file mode 100644
index 56caf65..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/BufferedOutputStream.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * This class is a slight modification of the
- * {@link java.io.BufferedOutputStream} class. This implementation differs in
- * that it does not mark methods as synchronized. This means that this class is
- * not suitable for writing by multiple concurrent threads. However, the removal
- * of the synchronized keyword results in potentially much better performance.
- */
-public class BufferedOutputStream extends FilterOutputStream {
-
- /**
- * The internal buffer where data is stored.
- */
- protected byte buf[];
-
- /**
- * The number of valid bytes in the buffer. This value is always in the
- * range <tt>0</tt> through <tt>buf.length</tt>; elements
- * <tt>buf[0]</tt> through <tt>buf[count-1]</tt> contain valid byte data.
- */
- protected int count;
-
- /**
- * Creates a new buffered output stream to write data to the specified
- * underlying output stream.
- *
- * @param out the underlying output stream.
- */
- public BufferedOutputStream(OutputStream out) {
- this(out, 8192);
- }
-
- /**
- * Creates a new buffered output stream to write data to the specified
- * underlying output stream with the specified buffer size.
- *
- * @param out the underlying output stream.
- * @param size the buffer size.
- * @exception IllegalArgumentException if size <= 0.
- */
- public BufferedOutputStream(OutputStream out, int size) {
- super(out);
- if (size <= 0) {
- throw new IllegalArgumentException("Buffer size <= 0");
- }
- buf = new byte[size];
- }
-
- /**
- * Flush the internal buffer
- */
- private void flushBuffer() throws IOException {
- if (count > 0) {
- out.write(buf, 0, count);
- count = 0;
- }
- }
-
- /**
- * Writes the specified byte to this buffered output stream.
- *
- * @param b the byte to be written.
- * @exception IOException if an I/O error occurs.
- */
- @Override
- public void write(int b) throws IOException {
- if (count >= buf.length) {
- flushBuffer();
- }
- buf[count++] = (byte) b;
- }
-
- /**
- * Writes <code>len</code> bytes from the specified byte array starting at
- * offset <code>off</code> to this buffered output stream.
- *
- * <p>
- * Ordinarily this method stores bytes from the given array into this
- * stream's buffer, flushing the buffer to the underlying output stream as
- * needed. If the requested length is at least as large as this stream's
- * buffer, however, then this method will flush the buffer and write the
- * bytes directly to the underlying output stream. Thus redundant
- * <code>BufferedOutputStream</code>s will not copy data unnecessarily.
- *
- * @param b the data.
- * @param off the start offset in the data.
- * @param len the number of bytes to write.
- * @exception IOException if an I/O error occurs.
- */
- @Override
- public void write(byte b[], int off, int len) throws IOException {
- if (len >= buf.length) {
- /* If the request length exceeds the size of the output buffer,
- flush the output buffer and then write the data directly.
- In this way buffered streams will cascade harmlessly. */
- flushBuffer();
- out.write(b, off, len);
- return;
- }
- if (len >= buf.length - count) {
- flushBuffer();
- }
- System.arraycopy(b, off, buf, count, len);
- count += len;
- }
-
- /**
- * Flushes this buffered output stream. This forces any buffered output
- * bytes to be written out to the underlying output stream.
- *
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public void flush() throws IOException {
- flushBuffer();
- out.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayInputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayInputStream.java
deleted file mode 100644
index 9defab4..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayInputStream.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.InputStream;
-
-/**
- * This class performs the same function as java.io.ByteArrayInputStream but
- * does not mark its methods as synchronized
- */
-public class ByteArrayInputStream extends InputStream {
-
- /**
- * An array of bytes that was provided by the creator of the stream.
- * Elements <code>buf[0]</code> through <code>buf[count-1]</code> are the
- * only bytes that can ever be read from the stream; element
- * <code>buf[pos]</code> is the next byte to be read.
- */
- protected byte buf[];
-
- /**
- * The index of the next character to read from the input stream buffer.
- * This value should always be nonnegative and not larger than the value of
- * <code>count</code>. The next byte to be read from the input stream buffer
- * will be <code>buf[pos]</code>.
- */
- protected int pos;
-
- /**
- * The currently marked position in the stream. ByteArrayInputStream objects
- * are marked at position zero by default when constructed. They may be
- * marked at another position within the buffer by the <code>mark()</code>
- * method. The current buffer position is set to this point by the
- * <code>reset()</code> method.
- * <p>
- * If no mark has been set, then the value of mark is the offset passed to
- * the constructor (or 0 if the offset was not supplied).
- *
- * @since JDK1.1
- */
- protected int mark = 0;
-
- /**
- * The index one greater than the last valid character in the input stream
- * buffer. This value should always be nonnegative and not larger than the
- * length of <code>buf</code>. It is one greater than the position of the
- * last byte within <code>buf</code> that can ever be read from the input
- * stream buffer.
- */
- protected int count;
-
- /**
- * Creates a <code>ByteArrayInputStream</code> so that it uses
- * <code>buf</code> as its buffer array. The buffer array is not copied. The
- * initial value of <code>pos</code> is <code>0</code> and the initial value
- * of <code>count</code> is the length of <code>buf</code>.
- *
- * @param buf the input buffer.
- */
- public ByteArrayInputStream(byte buf[]) {
- this.buf = buf;
- this.pos = 0;
- this.count = buf.length;
- }
-
- /**
- * Creates <code>ByteArrayInputStream</code> that uses <code>buf</code> as
- * its buffer array. The initial value of <code>pos</code> is
- * <code>offset</code> and the initial value of <code>count</code> is the
- * minimum of <code>offset+length</code> and <code>buf.length</code>. The
- * buffer array is not copied. The buffer's mark is set to the specified
- * offset.
- *
- * @param buf the input buffer.
- * @param offset the offset in the buffer of the first byte to read.
- * @param length the maximum number of bytes to read from the buffer.
- */
- public ByteArrayInputStream(byte buf[], int offset, int length) {
- this.buf = buf;
- this.pos = offset;
- this.count = Math.min(offset + length, buf.length);
- this.mark = offset;
- }
-
- /**
- * Reads the next byte of data from this input stream. The value byte is
- * returned as an <code>int</code> in the range <code>0</code> to
- * <code>255</code>. If no byte is available because the end of the stream
- * has been reached, the value <code>-1</code> is returned.
- * <p>
- * This <code>read</code> method cannot block.
- *
- * @return the next byte of data, or <code>-1</code> if the end of the
- * stream has been reached.
- */
- @Override
- public int read() {
- return (pos < count) ? (buf[pos++] & 0xff) : -1;
- }
-
- /**
- * Reads up to <code>len</code> bytes of data into an array of bytes from
- * this input stream. If <code>pos</code> equals <code>count</code>, then
- * <code>-1</code> is returned to indicate end of file. Otherwise, the
- * number <code>k</code> of bytes read is equal to the smaller of
- * <code>len</code> and <code>count-pos</code>. If <code>k</code> is
- * positive, then bytes <code>buf[pos]</code> through
- * <code>buf[pos+k-1]</code> are copied into <code>b[off]</code> through
- * <code>b[off+k-1]</code> in the manner performed by
- * <code>System.arraycopy</code>. The value <code>k</code> is added into
- * <code>pos</code> and <code>k</code> is returned.
- * <p>
- * This <code>read</code> method cannot block.
- *
- * @param b the buffer into which the data is read.
- * @param off the start offset in the destination array <code>b</code>
- * @param len the maximum number of bytes read.
- * @return the total number of bytes read into the buffer, or
- * <code>-1</code> if there is no more data because the end of the stream
- * has been reached.
- * @exception NullPointerException If <code>b</code> is <code>null</code>.
- * @exception IndexOutOfBoundsException If <code>off</code> is negative,
- * <code>len</code> is negative, or <code>len</code> is greater than
- * <code>b.length - off</code>
- */
- @Override
- public int read(byte b[], int off, int len) {
- if (b == null) {
- throw new NullPointerException();
- } else if (off < 0 || len < 0 || len > b.length - off) {
- throw new IndexOutOfBoundsException();
- }
-
- if (pos >= count) {
- return -1;
- }
-
- int avail = count - pos;
- if (len > avail) {
- len = avail;
- }
- if (len <= 0) {
- return 0;
- }
- System.arraycopy(buf, pos, b, off, len);
- pos += len;
- return len;
- }
-
- /**
- * Skips <code>n</code> bytes of input from this input stream. Fewer bytes
- * might be skipped if the end of the input stream is reached. The actual
- * number <code>k</code> of bytes to be skipped is equal to the smaller of
- * <code>n</code> and <code>count-pos</code>. The value <code>k</code> is
- * added into <code>pos</code> and <code>k</code> is returned.
- *
- * @param n the number of bytes to be skipped.
- * @return the actual number of bytes skipped.
- */
- @Override
- public long skip(long n) {
- long k = count - pos;
- if (n < k) {
- k = n < 0 ? 0 : n;
- }
-
- pos += k;
- return k;
- }
-
- /**
- * Returns the number of remaining bytes that can be read (or skipped over)
- * from this input stream.
- * <p>
- * The value returned is <code>count - pos</code>, which is the number
- * of bytes remaining to be read from the input buffer.
- *
- * @return the number of remaining bytes that can be read (or skipped over)
- * from this input stream without blocking.
- */
- @Override
- public int available() {
- return count - pos;
- }
-
- /**
- * Tests if this <code>InputStream</code> supports mark/reset. The
- * <code>markSupported</code> method of <code>ByteArrayInputStream</code>
- * always returns <code>true</code>.
- *
- * @since JDK1.1
- */
- @Override
- public boolean markSupported() {
- return true;
- }
-
- /**
- * Set the current marked position in the stream. ByteArrayInputStream
- * objects are marked at position zero by default when constructed. They may
- * be marked at another position within the buffer by this method.
- * <p>
- * If no mark has been set, then the value of the mark is the offset passed
- * to the constructor (or 0 if the offset was not supplied).
- *
- * <p>
- * Note: The <code>readAheadLimit</code> for this class has no meaning.
- *
- * @since JDK1.1
- */
- @Override
- public void mark(int readAheadLimit) {
- mark = pos;
- }
-
- /**
- * Resets the buffer to the marked position. The marked position is 0 unless
- * another position was marked or an offset was specified in the
- * constructor.
- */
- @Override
- public void reset() {
- pos = mark;
- }
-
- /**
- * Closing a <tt>ByteArrayInputStream</tt> has no effect. The methods in
- * this class can be called after the stream has been closed without
- * generating an <tt>IOException</tt>.
- * <p>
- */
- @Override
- public void close() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayOutputStream.java
deleted file mode 100644
index b504c23..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteArrayOutputStream.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-
-/**
- * This class provides a more efficient implementation of the
- * java.io.ByteArrayOutputStream. The efficiency is gained in two ways:
- * <ul>
- * <li>The write methods are not synchronized</li>
- * <li>The class provides {@link #getUnderlyingBuffer()} and
- * {@link #getBufferLength()}, which can be used to access the underlying byte
- * array directly, rather than the System.arraycopy that {@link #toByteArray()}
- * uses
- * </ul>
- *
- */
-public class ByteArrayOutputStream extends OutputStream {
-
- /**
- * The buffer where data is stored.
- */
- protected byte buf[];
-
- /**
- * The number of valid bytes in the buffer.
- */
- protected int count;
-
- /**
- * Creates a new byte array output stream. The buffer capacity is initially
- * 32 bytes, though its size increases if necessary.
- */
- public ByteArrayOutputStream() {
- this(32);
- }
-
- /**
- * Creates a new byte array output stream, with a buffer capacity of the
- * specified size, in bytes.
- *
- * @param size the initial size.
- * @exception IllegalArgumentException if size is negative.
- */
- public ByteArrayOutputStream(int size) {
- if (size < 0) {
- throw new IllegalArgumentException("Negative initial size: "
- + size);
- }
- buf = new byte[size];
- }
-
- /**
- * Increases the capacity if necessary to ensure that it can hold at least
- * the number of elements specified by the minimum capacity argument.
- *
- * @param minCapacity the desired minimum capacity
- * @throws OutOfMemoryError if {@code minCapacity < 0}. This is interpreted
- * as a request for the unsatisfiably large capacity
- * {@code (long) Integer.MAX_VALUE + (minCapacity - Integer.MAX_VALUE)}.
- */
- private void ensureCapacity(int minCapacity) {
- // overflow-conscious code
- if (minCapacity - buf.length > 0) {
- grow(minCapacity);
- }
- }
-
- /**
- * Increases the capacity to ensure that it can hold at least the number of
- * elements specified by the minimum capacity argument.
- *
- * @param minCapacity the desired minimum capacity
- */
- private void grow(int minCapacity) {
- // overflow-conscious code
- int oldCapacity = buf.length;
- int newCapacity = oldCapacity << 1;
- if (newCapacity - minCapacity < 0) {
- newCapacity = minCapacity;
- }
- if (newCapacity < 0) {
- if (minCapacity < 0) // overflow
- {
- throw new OutOfMemoryError();
- }
- newCapacity = Integer.MAX_VALUE;
- }
- buf = Arrays.copyOf(buf, newCapacity);
- }
-
- /**
- * Writes the specified byte to this byte array output stream.
- *
- * @param b the byte to be written.
- */
- @Override
- public void write(int b) {
- ensureCapacity(count + 1);
- buf[count] = (byte) b;
- count += 1;
- }
-
- /**
- * Writes <code>len</code> bytes from the specified byte array starting at
- * offset <code>off</code> to this byte array output stream.
- *
- * @param b the data.
- * @param off the start offset in the data.
- * @param len the number of bytes to write.
- */
- @Override
- public void write(byte b[], int off, int len) {
- if ((off < 0) || (off > b.length) || (len < 0)
- || ((off + len) - b.length > 0)) {
- throw new IndexOutOfBoundsException();
- }
- ensureCapacity(count + len);
- System.arraycopy(b, off, buf, count, len);
- count += len;
- }
-
- /**
- * Writes the complete contents of this byte array output stream to the
- * specified output stream argument, as if by calling the output stream's
- * write method using <code>out.write(buf, 0, count)</code>.
- *
- * @param out the output stream to which to write the data.
- * @exception IOException if an I/O error occurs.
- */
- public void writeTo(OutputStream out) throws IOException {
- out.write(buf, 0, count);
- }
-
- /**
- * Resets the <code>count</code> field of this byte array output stream to
- * zero, so that all currently accumulated output in the output stream is
- * discarded. The output stream can be used again, reusing the already
- * allocated buffer space.
- *
- * @see java.io.ByteArrayInputStream#count
- */
- public void reset() {
- count = 0;
- }
-
- /**
- * Creates a newly allocated byte array. Its size is the current size of
- * this output stream and the valid contents of the buffer have been copied
- * into it.
- *
- * @return the current contents of this output stream, as a byte array.
- * @see java.io.ByteArrayOutputStream#size()
- */
- public byte toByteArray ()
- [] {
- return Arrays.copyOf(buf, count);
- }
-
- /**
- * Returns the current size of the buffer.
- *
- * @return the value of the <code>count</code> field, which is the number of
- * valid bytes in this output stream.
- * @see java.io.ByteArrayOutputStream#count
- */
- public int size() {
- return count;
- }
-
- /**
- * Converts the buffer's contents into a string decoding bytes using the
- * platform's default character set. The length of the new <tt>String</tt>
- * is a function of the character set, and hence may not be equal to the
- * size of the buffer.
- *
- * <p>
- * This method always replaces malformed-input and unmappable-character
- * sequences with the default replacement string for the platform's default
- * character set. The {@linkplain java.nio.charset.CharsetDecoder} class
- * should be used when more control over the decoding process is required.
- *
- * @return String decoded from the buffer's contents.
- * @since JDK1.1
- */
- @Override
- public String toString() {
- return new String(buf, 0, count);
- }
-
- /**
- * Converts the buffer's contents into a string by decoding the bytes using
- * the specified {@link java.nio.charset.Charset charsetName}. The length of
- * the new <tt>String</tt> is a function of the charset, and hence may not
- * be equal to the length of the byte array.
- *
- * <p>
- * This method always replaces malformed-input and unmappable-character
- * sequences with this charset's default replacement string. The {@link
- * java.nio.charset.CharsetDecoder} class should be used when more control
- * over the decoding process is required.
- *
- * @param charsetName the name of a supported
- * {@linkplain java.nio.charset.Charset <code>charset</code>}
- * @return String decoded from the buffer's contents.
- * @exception UnsupportedEncodingException If the named charset is not
- * supported
- * @since JDK1.1
- */
- public String toString(String charsetName) throws UnsupportedEncodingException {
- return new String(buf, 0, count, charsetName);
- }
-
- /**
- * Closing a <tt>ByteArrayOutputStream</tt> has no effect. The methods in
- * this class can be called after the stream has been closed without
- * generating an <tt>IOException</tt>.
- * <p>
- *
- */
- @Override
- public void close() {
- }
-
- public byte[] getUnderlyingBuffer() {
- return buf;
- }
-
- public int getBufferLength() {
- return count;
- }
-}
[29/31] incubator-nifi git commit: NIFI-192 fixed dependency
inclusion/exclusion logic for RPM
Posted by mc...@apache.org.
NIFI-192 fixed dependency inclusion/exclusion logic for RPM
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/42f9cfb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/42f9cfb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/42f9cfb2
Branch: refs/heads/NIFI-182
Commit: 42f9cfb2d94519fe11c83a7a28ce8ca2bdf896e1
Parents: a3664a0
Author: joewitt <jo...@apache.org>
Authored: Sat Dec 20 21:43:09 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sat Dec 20 21:43:09 2014 -0500
----------------------------------------------------------------------
assembly/pom.xml | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/42f9cfb2/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 45ac535..7f7f695 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -383,9 +383,6 @@
<provides>
<provide>nifi</provide>
</provides>
- <requires>
- <require>jdk</require>
- </requires>
<mappings>
<mapping>
<directory>/opt/nifi/nifi-${project.version}</directory>
@@ -428,7 +425,19 @@
</mapping>
<mapping>
<directory>/opt/nifi/nifi-${project.version}/lib</directory>
- <dependency />
+ <dependency>
+ <excludes>
+ <exclude>org.apache.nifi:nifi-bootstrap</exclude>
+ </excludes>
+ </dependency>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory>
+ <dependency>
+ <includes>
+ <include>org.apache.nifi:nifi-bootstrap</include>
+ </includes>
+ </dependency>
</mapping>
<mapping>
<directory>/opt/nifi/nifi-${project.version}/docs</directory>
[14/31] incubator-nifi git commit: NIFI-150 fixed error with
inserting license header - still need to figure out the license header
addition
Posted by mc...@apache.org.
NIFI-150 fixed error with inserting license header - still need to figure out the license header addition
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/4031bd54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/4031bd54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/4031bd54
Branch: refs/heads/NIFI-182
Commit: 4031bd5495bd9ac62a3c3d8da2d3e9083d59fdfa
Parents: b0a5115
Author: joewitt <jo...@apache.org>
Authored: Fri Dec 19 14:57:13 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Fri Dec 19 14:57:13 2014 -0500
----------------------------------------------------------------------
nifi-docs/pom.xml | 29 +++++++++++++++--------------
1 file changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4031bd54/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
index 79ae0ac..bfa9e44 100644
--- a/nifi-docs/pom.xml
+++ b/nifi-docs/pom.xml
@@ -49,6 +49,7 @@
</execution>
</executions>
<configuration>
+ <ignoreErrors>true</ignoreErrors>
<file>target/generated-docs/nifi-user-guide.html</file>
<regex>true</regex>
<regexFlags>
@@ -58,22 +59,22 @@
<token>^(.*)$</token>
<value>
<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-$1
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ $1
</value>
</configuration>
</plugin>
[06/31] incubator-nifi git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-nifi into develop
Posted by mc...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-nifi into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/ea17562d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/ea17562d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/ea17562d
Branch: refs/heads/NIFI-182
Commit: ea17562dd328d5f38e11ca8ff4eb81a617fb8e6e
Parents: cbb1746 732cb70
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 10:03:28 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 10:03:28 2014 -0500
----------------------------------------------------------------------
nar-bundles/framework-bundle/framework/web/nifi-web-api/pom.xml | 2 +-
.../nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp | 2 +-
nar-bundles/pom.xml | 3 +++
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[19/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/search-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
----------------------------------------------------------------------
diff --git a/commons/search-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java b/commons/search-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
deleted file mode 100644
index 62de964..0000000
--- a/commons/search-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.search;
-
-import java.nio.charset.Charset;
-import java.util.Arrays;
-
-/**
- * This is an immutable thread safe object representing a search term
- *
- * @author
- * @param <T>
- */
-public class SearchTerm<T> {
-
- private final byte[] bytes;
- private final int hashCode;
- private final T reference;
-
- /**
- * Constructs a SearchTerm. Defensively copies the given byte array
- *
- * @param bytes
- * @throws IllegalArgument exception if given bytes are null or 0 length
- */
- public SearchTerm(final byte[] bytes) {
- this(bytes, true, null);
- }
-
- /**
- * Constructs a search term. Optionally performs a defensive copy of the
- * given byte array. If the caller indicates a defensive copy is not
- * necessary then they must not change the given arrays state any longer
- *
- * @param bytes
- * @param defensiveCopy
- * @param reference
- */
- public SearchTerm(final byte[] bytes, final boolean defensiveCopy, final T reference) {
- if (bytes == null || bytes.length == 0) {
- throw new IllegalArgumentException();
- }
- if (defensiveCopy) {
- this.bytes = Arrays.copyOf(bytes, bytes.length);
- } else {
- this.bytes = bytes;
- }
- this.hashCode = Arrays.hashCode(this.bytes);
- this.reference = reference;
- }
-
- public int get(final int index) {
- return bytes[index] & 0xff;
- }
-
- /**
- * @return size in of search term in bytes
- */
- public int size() {
- return bytes.length;
- }
-
- /**
- * @return reference object for this given search term
- */
- public T getReference() {
- return reference;
- }
-
- /**
- * Determines if the given window starts with the same bytes as this term
- *
- * @param window Current window of bytes from the haystack being evaluated.
- * @param windowLength The length of the window to consider
- * @return true if this term starts with the same bytes of the given window
- */
- public boolean startsWith(byte[] window, int windowLength) {
- if (windowLength > window.length) {
- throw new IndexOutOfBoundsException();
- }
- if (bytes.length < windowLength) {
- return false;
- }
- for (int i = 0; i < bytes.length && i < windowLength; i++) {
- if (bytes[i] != window[i]) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * @return a defensive copy of the internal byte structure
- */
- public byte[] getBytes() {
- return Arrays.copyOf(bytes, bytes.length);
- }
-
- @Override
- public int hashCode() {
- return hashCode;
- }
-
- @Override
- public boolean equals(final Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final SearchTerm other = (SearchTerm) obj;
- if (this.hashCode != other.hashCode) {
- return false;
- }
- return Arrays.equals(this.bytes, other.bytes);
- }
-
- @Override
- public String toString() {
- return new String(bytes);
- }
-
- public String toString(final Charset charset) {
- return new String(bytes, charset);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
----------------------------------------------------------------------
diff --git a/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java b/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
deleted file mode 100644
index 3b8afaf..0000000
--- a/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.search.ahocorasick;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Set;
-
-import org.apache.nifi.util.search.Search;
-import org.apache.nifi.util.search.SearchTerm;
-
-public class AhoCorasick<T> implements Search<T> {
-
- private Node root = null;
-
- /**
- * Constructs a new search object.
- *
- * @throws IllegalArgumentException if given terms are null or empty
- */
- public AhoCorasick() {
- }
-
- @Override
- public void initializeDictionary(final Set<SearchTerm<T>> terms) {
- if (root != null) {
- throw new IllegalStateException();
- }
- root = new Node();
- if (terms == null || terms.isEmpty()) {
- throw new IllegalArgumentException();
- }
- for (final SearchTerm<T> term : terms) {
- int i = 0;
- Node nextNode = root;
- while (true) {
- nextNode = addMatch(term, i, nextNode);
- if (nextNode == null) {
- break; //we're done
- }
- i++;
- }
- }
- initialize();
- }
-
- private Node addMatch(final SearchTerm<T> term, final int offset, final Node current) {
- final int index = term.get(offset);
- boolean atEnd = (offset == (term.size() - 1));
- if (current.getNeighbor(index) == null) {
- if (atEnd) {
- current.setNeighbor(new Node(term), index);
- return null;
- }
- current.setNeighbor(new Node(), index);
- } else if (atEnd) {
- current.getNeighbor(index).setMatchingTerm(term);
- return null;
- }
- return current.getNeighbor(index);
- }
-
- private void initialize() {
- //perform bgs to build failure links
- final Queue<Node> queue = new LinkedList<>();
- queue.add(root);
- root.setFailureNode(null);
- while (!queue.isEmpty()) {
- final Node current = queue.poll();
- for (int i = 0; i < 256; i++) {
- final Node next = current.getNeighbor(i);
- if (next != null) {
- //traverse failure to get state
- Node fail = current.getFailureNode();
- while ((fail != null) && fail.getNeighbor(i) == null) {
- fail = fail.getFailureNode();
- }
- if (fail != null) {
- next.setFailureNode(fail.getNeighbor(i));
- } else {
- next.setFailureNode(root);
- }
- queue.add(next);
- }
- }
- }
- }
-
- @Override
- public SearchState search(final InputStream stream, final boolean findAll) throws IOException {
- return search(stream, findAll, null);
- }
-
- private SearchState search(final InputStream stream, final boolean findAll, final SearchState state) throws IOException {
- if (root == null) {
- throw new IllegalStateException();
- }
- final SearchState<T> currentState = (state == null) ? new SearchState(root) : state;
- if (!findAll && currentState.foundMatch()) {
- throw new IllegalStateException("A match has already been found yet we're being asked to keep searching");
- }
- Node current = currentState.getCurrentNode();
- int currentChar;
- while ((currentChar = stream.read()) >= 0) {
- currentState.incrementBytesRead(1L);
- Node next = current.getNeighbor(currentChar);
- if (next == null) {
- next = current.getFailureNode();
- while ((next != null) && next.getNeighbor(currentChar) == null) {
- next = next.getFailureNode();
- }
- if (next != null) {
- next = next.getNeighbor(currentChar);
- } else {
- next = root;
- }
- }
- if (next == null) {
- throw new IllegalStateException("tree out of sync");
- }
- //Accept condition
- if (next.hasMatch()) {
- currentState.addResult(next.getMatchingTerm());
- }
- for (Node failNode = next.getFailureNode(); failNode != null; failNode = failNode.getFailureNode()) {
- if (failNode.hasMatch()) {
- currentState.addResult(failNode.getMatchingTerm());
- }
- }
- current = next;
- if (currentState.foundMatch() && !findAll) {
- break;//give up as soon as we have at least one match
- }
- }
- currentState.setCurrentNode(current);
- return currentState;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
----------------------------------------------------------------------
diff --git a/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java b/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
deleted file mode 100644
index 0ac325c..0000000
--- a/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.search.ahocorasick;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.nifi.util.search.SearchTerm;
-
-/**
- *
- * @author
- */
-public class Node {
-
- private final Map<Integer, Node> neighborMap;
- private Node failureNode;
- private SearchTerm<?> term;
-
- Node(final SearchTerm<?> term) {
- this();
- this.term = term;
- }
-
- Node() {
- neighborMap = new HashMap<>();
- term = null;
- }
-
- void setFailureNode(final Node fail) {
- failureNode = fail;
- }
-
- public Node getFailureNode() {
- return failureNode;
- }
-
- public boolean hasMatch() {
- return term != null;
- }
-
- void setMatchingTerm(final SearchTerm<?> term) {
- this.term = term;
- }
-
- public SearchTerm<?> getMatchingTerm() {
- return term;
- }
-
- public Node getNeighbor(final int index) {
- return neighborMap.get(index);
- }
-
- void setNeighbor(final Node neighbor, final int index) {
- neighborMap.put(index, neighbor);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
----------------------------------------------------------------------
diff --git a/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java b/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
deleted file mode 100644
index 6d36ad0..0000000
--- a/commons/search-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.search.ahocorasick;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.nifi.util.search.SearchTerm;
-
-public class SearchState<T> {
-
- private Node currentNode;
- private final Map<SearchTerm<T>, List<Long>> resultMap;
- private long bytesRead;
-
- SearchState(final Node rootNode) {
- resultMap = new HashMap<>(5);
- currentNode = rootNode;
- bytesRead = 0L;
- }
-
- void incrementBytesRead(final long increment) {
- bytesRead += increment;
- }
-
- void setCurrentNode(final Node curr) {
- currentNode = curr;
- }
-
- public Node getCurrentNode() {
- return currentNode;
- }
-
- public Map<SearchTerm<T>, List<Long>> getResults() {
- return new HashMap<>(resultMap);
- }
-
- void addResult(final SearchTerm matchingTerm) {
- final List<Long> indexes = (resultMap.containsKey(matchingTerm)) ? resultMap.get(matchingTerm) : new ArrayList<Long>(5);
- indexes.add(bytesRead);
- resultMap.put(matchingTerm, indexes);
- }
-
- public boolean foundMatch() {
- return !resultMap.isEmpty();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/wali/pom.xml
----------------------------------------------------------------------
diff --git a/commons/wali/pom.xml b/commons/wali/pom.xml
index 5a06a3f..347c8cc 100644
--- a/commons/wali/pom.xml
+++ b/commons/wali/pom.xml
@@ -35,7 +35,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/wali/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
----------------------------------------------------------------------
diff --git a/commons/wali/src/main/java/org/wali/MinimalLockingWriteAheadLog.java b/commons/wali/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
index 95cf4da..19208d3 100644
--- a/commons/wali/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
+++ b/commons/wali/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
@@ -58,8 +58,8 @@ import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/cluster/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/cluster/pom.xml b/nar-bundles/framework-bundle/framework/cluster/pom.xml
index 78f4527..6712802 100644
--- a/nar-bundles/framework-bundle/framework/cluster/pom.xml
+++ b/nar-bundles/framework-bundle/framework/cluster/pom.xml
@@ -42,7 +42,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-file-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewall.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewall.java b/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewall.java
index bcee661..916ec14 100644
--- a/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewall.java
+++ b/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewall.java
@@ -22,7 +22,7 @@ import java.net.UnknownHostException;
import java.util.*;
import org.apache.commons.net.util.SubnetUtils;
import org.apache.nifi.cluster.firewall.ClusterNodeFirewall;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.logging.NiFiLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/flow/impl/DataFlowDaoImpl.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/flow/impl/DataFlowDaoImpl.java b/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/flow/impl/DataFlowDaoImpl.java
index 702f081..b7b142b 100644
--- a/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/flow/impl/DataFlowDaoImpl.java
+++ b/nar-bundles/framework-bundle/framework/cluster/src/main/java/org/apache/nifi/cluster/flow/impl/DataFlowDaoImpl.java
@@ -48,11 +48,11 @@ import org.apache.nifi.cluster.protocol.DataFlow;
import org.apache.nifi.cluster.protocol.NodeIdentifier;
import org.apache.nifi.cluster.protocol.StandardDataFlow;
import org.apache.nifi.cluster.protocol.jaxb.message.NodeIdentifierAdapter;
-import org.apache.nifi.file.FileUtils;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.ByteArrayInputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.util.file.FileUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.NiFiLog;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/cluster/src/test/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewallTest.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/cluster/src/test/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewallTest.java b/nar-bundles/framework-bundle/framework/cluster/src/test/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewallTest.java
index 2fcf7ef..e5db7ca 100644
--- a/nar-bundles/framework-bundle/framework/cluster/src/test/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewallTest.java
+++ b/nar-bundles/framework-bundle/framework/cluster/src/test/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewallTest.java
@@ -16,10 +16,9 @@
*/
package org.apache.nifi.cluster.firewall.impl;
-import org.apache.nifi.cluster.firewall.impl.FileBasedClusterNodeFirewall;
import java.io.File;
import java.io.IOException;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.junit.After;
import org.junit.Before;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core-api/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core-api/pom.xml b/nar-bundles/framework-bundle/framework/core-api/pom.xml
index b163cd0..63645f3 100644
--- a/nar-bundles/framework-bundle/framework/core-api/pom.xml
+++ b/nar-bundles/framework-bundle/framework/core-api/pom.xml
@@ -30,7 +30,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>remote-communications-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
@@ -49,10 +49,6 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-utils</artifactId>
- </dependency>
- <dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/pom.xml b/nar-bundles/framework-bundle/framework/core/pom.xml
index 1836d32..47b52ea 100644
--- a/nar-bundles/framework-bundle/framework/core/pom.xml
+++ b/nar-bundles/framework-bundle/framework/core/pom.xml
@@ -42,10 +42,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-file-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
@@ -53,10 +49,6 @@
<artifactId>site-to-site</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
- </dependency>
- <dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
index ad95f8e..4b8d43b 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
@@ -62,7 +62,7 @@ import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ContentClaimManager;
import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.events.EventReporter;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.reporting.Severity;
import org.apache.nifi.util.FormatUtils;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowController.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowController.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowController.java
index 545017a..e6d67a0 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowController.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowController.java
@@ -125,7 +125,7 @@ import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor;
import org.apache.nifi.groups.StandardProcessGroup;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.logging.LogRepository;
import org.apache.nifi.logging.LogRepositoryFactory;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowUnmarshaller.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowUnmarshaller.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowUnmarshaller.java
index fa33b49..42d7f1c 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowUnmarshaller.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/FlowUnmarshaller.java
@@ -26,7 +26,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.nifi.encrypt.StringEncryptor;
-import org.apache.nifi.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
import org.apache.nifi.web.api.dto.FlowSnippetDTO;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/SnippetManager.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/SnippetManager.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/SnippetManager.java
index 415472f..3a9662e 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/SnippetManager.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/SnippetManager.java
@@ -25,10 +25,10 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.nifi.io.ByteArrayInputStream;
-import org.apache.nifi.io.ByteArrayOutputStream;
-import org.apache.nifi.io.DataOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.persistence.StandardSnippetDeserializer;
import org.apache.nifi.persistence.StandardSnippetSerializer;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowService.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowService.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowService.java
index 1443737..d459b00 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowService.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowService.java
@@ -59,7 +59,7 @@ import org.apache.nifi.cluster.protocol.message.ReconnectionRequestMessage;
import org.apache.nifi.cluster.protocol.message.ReconnectionResponseMessage;
import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.events.BulletinFactory;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.lifecycle.LifeCycleStartException;
import org.apache.nifi.logging.LogLevel;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
index 17a1702..ffea644 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
@@ -52,7 +52,7 @@ import org.apache.nifi.connectable.Size;
import org.apache.nifi.controller.exception.ProcessorInstantiationException;
import org.apache.nifi.controller.label.Label;
import org.apache.nifi.events.BulletinFactory;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.fingerprint.FingerprintException;
import org.apache.nifi.fingerprint.FingerprintFactory;
import org.apache.nifi.flowfile.FlowFilePrioritizer;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/TemplateManager.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/TemplateManager.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/TemplateManager.java
index 68acc2f..aa095d1 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/TemplateManager.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/TemplateManager.java
@@ -42,10 +42,10 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.nifi.io.ByteArrayInputStream;
-import org.apache.nifi.io.ByteArrayOutputStream;
-import org.apache.nifi.io.DataOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.persistence.TemplateDeserializer;
import org.apache.nifi.persistence.TemplateSerializer;
import org.apache.nifi.web.api.dto.ConnectableDTO;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
index 5fbbfd5..a3e24c4 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
@@ -65,8 +65,8 @@ import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ContentClaimManager;
import org.apache.nifi.controller.repository.io.SyncOnCloseOutputStream;
import org.apache.nifi.engine.FlowEngine;
-import org.apache.nifi.file.FileUtils;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.util.file.FileUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.util.LongHolder;
import org.apache.nifi.util.NiFiProperties;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 60dcdb3..3d3e854 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -53,9 +53,9 @@ import org.apache.nifi.controller.repository.io.LimitedInputStream;
import org.apache.nifi.controller.repository.io.LongHolder;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.NonCloseableInputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.NonCloseableInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.ProcessSession;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
index e39436e..8deda3c 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
@@ -44,8 +44,8 @@ import org.apache.nifi.controller.repository.claim.StandardContentClaim;
import org.apache.nifi.controller.repository.io.ArrayManagedOutputStream;
import org.apache.nifi.controller.repository.io.MemoryManager;
import org.apache.nifi.engine.FlowEngine;
-import org.apache.nifi.io.ByteArrayInputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.util.NiFiProperties;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java
index 5afb239..42bd55f 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java
@@ -36,7 +36,7 @@ import javax.xml.parsers.ParserConfigurationException;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.util.DomUtils;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/persistence/StandardXMLFlowConfigurationDAO.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/persistence/StandardXMLFlowConfigurationDAO.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/persistence/StandardXMLFlowConfigurationDAO.java
index cabb272..c11aa72 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/persistence/StandardXMLFlowConfigurationDAO.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/persistence/StandardXMLFlowConfigurationDAO.java
@@ -57,7 +57,7 @@ import org.apache.nifi.controller.reporting.StandardReportingInitializationConte
import org.apache.nifi.controller.service.ControllerServiceLoader;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.encrypt.StringEncryptor;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.nar.NarCloseable;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.reporting.ReportingInitializationContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
index d3fb41f..f214728 100644
--- a/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
+++ b/nar-bundles/framework-bundle/framework/core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
@@ -72,7 +72,7 @@ import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.groups.ProcessGroupCounts;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.remote.exception.BadRequestException;
import org.apache.nifi.remote.exception.HandshakeException;
import org.apache.nifi.remote.exception.PortNotRunningException;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java b/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
index 82c052b..054ef5e 100644
--- a/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
+++ b/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
@@ -35,7 +35,7 @@ import java.util.List;
import org.apache.nifi.connectable.Connection;
import org.apache.nifi.controller.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.StandardContentClaimManager;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/io/TestLimitedOutputStream.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/io/TestLimitedOutputStream.java b/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/io/TestLimitedOutputStream.java
index f13a752..da0c414 100644
--- a/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/io/TestLimitedOutputStream.java
+++ b/nar-bundles/framework-bundle/framework/core/src/test/java/org/apache/nifi/controller/repository/io/TestLimitedOutputStream.java
@@ -16,13 +16,12 @@
*/
package org.apache.nifi.controller.repository.io;
-import org.apache.nifi.controller.repository.io.LimitedInputStream;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.nifi.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/file-authorization-provider/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/file-authorization-provider/pom.xml b/nar-bundles/framework-bundle/framework/file-authorization-provider/pom.xml
index cb01488..c96f739 100644
--- a/nar-bundles/framework-bundle/framework/file-authorization-provider/pom.xml
+++ b/nar-bundles/framework-bundle/framework/file-authorization-provider/pom.xml
@@ -62,7 +62,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-file-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/file-authorization-provider/src/main/java/org/apache/nifi/authorization/FileAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/file-authorization-provider/src/main/java/org/apache/nifi/authorization/FileAuthorizationProvider.java b/nar-bundles/framework-bundle/framework/file-authorization-provider/src/main/java/org/apache/nifi/authorization/FileAuthorizationProvider.java
index 0f4a75c..beedd8e 100644
--- a/nar-bundles/framework-bundle/framework/file-authorization-provider/src/main/java/org/apache/nifi/authorization/FileAuthorizationProvider.java
+++ b/nar-bundles/framework-bundle/framework/file-authorization-provider/src/main/java/org/apache/nifi/authorization/FileAuthorizationProvider.java
@@ -36,7 +36,7 @@ import org.apache.nifi.authorization.exception.AuthorityAccessException;
import org.apache.nifi.authorization.exception.IdentityAlreadyExistsException;
import org.apache.nifi.authorization.exception.ProviderCreationException;
import org.apache.nifi.authorization.exception.UnknownIdentityException;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.user.generated.ObjectFactory;
import org.apache.nifi.user.generated.Role;
import org.apache.nifi.user.generated.User;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/file-authorization-provider/src/test/java/org/apache/nifi/authorization/FileAuthorizationProviderTest.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/file-authorization-provider/src/test/java/org/apache/nifi/authorization/FileAuthorizationProviderTest.java b/nar-bundles/framework-bundle/framework/file-authorization-provider/src/test/java/org/apache/nifi/authorization/FileAuthorizationProviderTest.java
index 3d0196d..d02d4d7 100644
--- a/nar-bundles/framework-bundle/framework/file-authorization-provider/src/test/java/org/apache/nifi/authorization/FileAuthorizationProviderTest.java
+++ b/nar-bundles/framework-bundle/framework/file-authorization-provider/src/test/java/org/apache/nifi/authorization/FileAuthorizationProviderTest.java
@@ -20,7 +20,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.nifi.authorization.exception.ProviderCreationException;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.util.NiFiProperties;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/site-to-site/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/site-to-site/pom.xml b/nar-bundles/framework-bundle/framework/site-to-site/pom.xml
index 30cd325..81a9146 100644
--- a/nar-bundles/framework-bundle/framework/site-to-site/pom.xml
+++ b/nar-bundles/framework-bundle/framework/site-to-site/pom.xml
@@ -31,10 +31,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-administration</artifactId>
</dependency>
<dependency>
@@ -50,10 +46,6 @@
<artifactId>core-api</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- </dependency>
- <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
@@ -74,10 +66,6 @@
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>remote-communications-utils</artifactId>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
index 59feb18..d18a4ee 100644
--- a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
+++ b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
@@ -27,7 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
index de27343..9e451fd 100644
--- a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
+++ b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.SocketChannel;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.ByteCountingInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.ByteCountingInputStream;
import org.apache.nifi.remote.io.InterruptableInputStream;
import org.apache.nifi.remote.protocol.CommunicationsInput;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
index 80c338c..26c0164 100644
--- a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
+++ b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.channels.SocketChannel;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.ByteCountingOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteCountingOutputStream;
import org.apache.nifi.remote.io.InterruptableOutputStream;
import org.apache.nifi.remote.protocol.CommunicationsOutput;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
index f7f5bf9..60ef33f 100644
--- a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
+++ b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
@@ -19,8 +19,8 @@ package org.apache.nifi.remote.io.socket.ssl;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.ByteCountingInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.ByteCountingInputStream;
import org.apache.nifi.remote.protocol.CommunicationsInput;
public class SSLSocketChannelInput implements CommunicationsInput {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
index 76d714a..dc3d68f 100644
--- a/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
+++ b/nar-bundles/framework-bundle/framework/site-to-site/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
@@ -19,8 +19,8 @@ package org.apache.nifi.remote.io.socket.ssl;
import java.io.IOException;
import java.io.OutputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.ByteCountingOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteCountingOutputStream;
import org.apache.nifi.remote.protocol.CommunicationsOutput;
public class SSLSocketChannelOutput implements CommunicationsOutput {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java b/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java
index 50bfbeb..2da7828 100644
--- a/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java
+++ b/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java
@@ -54,7 +54,7 @@ import org.apache.nifi.cluster.manager.impl.WebClusterManager;
import org.apache.nifi.cluster.node.Node;
import org.apache.nifi.cluster.protocol.NodeIdentifier;
import org.apache.nifi.controller.repository.claim.ContentDirection;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.NiFiServiceFacade;
import static org.apache.nifi.web.api.ApplicationResource.CLIENT_ID;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ClientResponseUtils.java
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ClientResponseUtils.java b/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ClientResponseUtils.java
index 70ea7d6..6a587aa 100644
--- a/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ClientResponseUtils.java
+++ b/nar-bundles/framework-bundle/framework/web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ClientResponseUtils.java
@@ -19,7 +19,7 @@ package org.apache.nifi.web.util;
import com.sun.jersey.api.client.ClientResponse;
import java.io.BufferedInputStream;
import java.io.IOException;
-import org.apache.nifi.io.NullOutputStream;
+import org.apache.nifi.stream.io.NullOutputStream;
import org.apache.nifi.logging.NiFiLog;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/pom.xml b/nar-bundles/hadoop-bundle/hdfs-processors/pom.xml
index 9a4bf2b..0325535 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/pom.xml
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/pom.xml
@@ -32,7 +32,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
@@ -42,10 +42,6 @@
<groupId>org.apache.nifi</groupId>
<artifactId>flowfile-packager</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FlowFileStreamUnpackerSequenceFileWriter.java
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FlowFileStreamUnpackerSequenceFileWriter.java b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FlowFileStreamUnpackerSequenceFileWriter.java
index 28386d8..d3fb97f 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FlowFileStreamUnpackerSequenceFileWriter.java
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/FlowFileStreamUnpackerSequenceFileWriter.java
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.processors.hadoop.util.InputStreamWritable;
import org.apache.nifi.util.FlowFilePackagerV3;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/PutHDFS.java
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/PutHDFS.java b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/PutHDFS.java
index be4ebac..5768da0 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/PutHDFS.java
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/PutHDFS.java
@@ -33,8 +33,8 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/SequenceFileWriterImpl.java
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/SequenceFileWriterImpl.java b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/SequenceFileWriterImpl.java
index 6eaad9f..4bb9ca9 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/SequenceFileWriterImpl.java
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/SequenceFileWriterImpl.java
@@ -23,7 +23,7 @@ import java.io.UnsupportedEncodingException;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processors.hadoop.util.ByteFilteringOutputStream;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/TarUnpackerSequenceFileWriter.java
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/TarUnpackerSequenceFileWriter.java b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/TarUnpackerSequenceFileWriter.java
index 2131eb8..82e1de2 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/TarUnpackerSequenceFileWriter.java
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/TarUnpackerSequenceFileWriter.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.io.InputStream;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.processors.hadoop.util.InputStreamWritable;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/ZipUnpackerSequenceFileWriter.java
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/ZipUnpackerSequenceFileWriter.java b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/ZipUnpackerSequenceFileWriter.java
index 476ceb3..c986e9a 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/ZipUnpackerSequenceFileWriter.java
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/ZipUnpackerSequenceFileWriter.java
@@ -23,7 +23,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.processors.hadoop.util.InputStreamWritable;
import org.apache.hadoop.io.SequenceFile.Writer;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/util/OutputStreamWritable.java
----------------------------------------------------------------------
diff --git a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/util/OutputStreamWritable.java b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/util/OutputStreamWritable.java
index 9773eb7..62fdc35 100644
--- a/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/util/OutputStreamWritable.java
+++ b/nar-bundles/hadoop-bundle/hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/util/OutputStreamWritable.java
@@ -21,8 +21,8 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.OutputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.DataOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/monitor-threshold-bundle/processor/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/monitor-threshold-bundle/processor/pom.xml b/nar-bundles/monitor-threshold-bundle/processor/pom.xml
index ad3b014..256f878 100644
--- a/nar-bundles/monitor-threshold-bundle/processor/pom.xml
+++ b/nar-bundles/monitor-threshold-bundle/processor/pom.xml
@@ -34,14 +34,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-file-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/monitor-threshold-bundle/processor/src/main/java/org/apache/nifi/processors/monitor/MonitorThreshold.java
----------------------------------------------------------------------
diff --git a/nar-bundles/monitor-threshold-bundle/processor/src/main/java/org/apache/nifi/processors/monitor/MonitorThreshold.java b/nar-bundles/monitor-threshold-bundle/processor/src/main/java/org/apache/nifi/processors/monitor/MonitorThreshold.java
index 7373380..ec07108 100644
--- a/nar-bundles/monitor-threshold-bundle/processor/src/main/java/org/apache/nifi/processors/monitor/MonitorThreshold.java
+++ b/nar-bundles/monitor-threshold-bundle/processor/src/main/java/org/apache/nifi/processors/monitor/MonitorThreshold.java
@@ -42,7 +42,7 @@ import java.util.regex.Pattern;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.file.FileUtils;
+import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/monitor-threshold-bundle/ui/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/monitor-threshold-bundle/ui/pom.xml b/nar-bundles/monitor-threshold-bundle/ui/pom.xml
index 645e4d6..af63349 100644
--- a/nar-bundles/monitor-threshold-bundle/ui/pom.xml
+++ b/nar-bundles/monitor-threshold-bundle/ui/pom.xml
@@ -123,7 +123,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-file-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/pom.xml b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/pom.xml
index 12e06e5..ad5cee3 100644
--- a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/pom.xml
+++ b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/pom.xml
@@ -37,10 +37,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
@@ -48,10 +44,6 @@
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
----------------------------------------------------------------------
diff --git a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
index a7ae850..5e4744b 100644
--- a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
+++ b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
@@ -26,8 +26,8 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import org.apache.nifi.io.ByteCountingInputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.ByteCountingInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.provenance.serialization.RecordReader;
public class StandardRecordReader implements RecordReader {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
----------------------------------------------------------------------
diff --git a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
index a70af9b..df93084 100644
--- a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
+++ b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
@@ -25,9 +25,9 @@ import java.util.UUID;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.ByteCountingOutputStream;
-import org.apache.nifi.io.DataOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteCountingOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
import org.apache.nifi.provenance.serialization.RecordWriter;
public class StandardRecordWriter implements RecordWriter {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/rollover/CompressionAction.java
----------------------------------------------------------------------
diff --git a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/rollover/CompressionAction.java b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/rollover/CompressionAction.java
index 0f5e9ce..d014618 100644
--- a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/rollover/CompressionAction.java
+++ b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/rollover/CompressionAction.java
@@ -22,8 +22,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import org.apache.nifi.io.GZIPOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.GZIPOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.provenance.lucene.IndexingAction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
----------------------------------------------------------------------
diff --git a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
index 198e018..f902b92 100644
--- a/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
+++ b/nar-bundles/persistent-provenance-repository-bundle/persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
@@ -26,7 +26,7 @@ import java.nio.file.Path;
import java.util.Collection;
import java.util.zip.GZIPInputStream;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.provenance.StandardRecordReader;
import org.apache.nifi.provenance.lucene.LuceneUtil;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/jms-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/pom.xml b/nar-bundles/standard-bundle/jms-processors/pom.xml
index 7dae21d..7cc7b17 100644
--- a/nar-bundles/standard-bundle/jms-processors/pom.xml
+++ b/nar-bundles/standard-bundle/jms-processors/pom.xml
@@ -36,7 +36,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>javax.jms</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/JmsConsumer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/JmsConsumer.java b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/JmsConsumer.java
index 29e62c6..24756c4 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/JmsConsumer.java
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/JmsConsumer.java
@@ -45,7 +45,7 @@ import javax.jms.MessageConsumer;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/PutJMS.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/PutJMS.java b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/PutJMS.java
index 3aae5cd..0624238 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/PutJMS.java
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/PutJMS.java
@@ -71,7 +71,7 @@ import javax.jms.StreamMessage;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
index 1f492e8..9a8da62 100644
--- a/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
+++ b/nar-bundles/standard-bundle/jms-processors/src/main/java/org/apache/nifi/processors/jms/util/JmsFactory.java
@@ -60,7 +60,7 @@ import javax.jms.StreamMessage;
import javax.jms.TextMessage;
import javax.jms.Topic;
-import org.apache.nifi.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
import org.apache.nifi.processor.ProcessContext;
import org.apache.activemq.ActiveMQConnectionFactory;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/pom.xml b/nar-bundles/standard-bundle/standard-processors/pom.xml
index 6e08598..f4cd2a0 100644
--- a/nar-bundles/standard-bundle/standard-processors/pom.xml
+++ b/nar-bundles/standard-bundle/standard-processors/pom.xml
@@ -37,11 +37,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
@@ -53,14 +49,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>naive-search-ring-buffer</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-search-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>distributed-cache-client-service-api</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
index bc0d230..21dfe93 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
@@ -35,9 +35,9 @@ import lzma.streams.LzmaOutputStream;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.GZIPOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.GZIPOutputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
index 861b515..eb079bb 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
@@ -23,7 +23,7 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.annotation.CapabilityDescription;
import org.apache.nifi.processor.annotation.EventDriven;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
index 704ec33..7697d06 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
@@ -31,7 +31,7 @@ import java.util.regex.Pattern;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java
index 2f3f34b..a1fc86d 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java
@@ -57,8 +57,8 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java
index d3c1347..8b4ce09 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java
@@ -54,8 +54,8 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
[20/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/test/resources/logback-test.xml b/commons/nifi-utils/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8651d47
--- /dev/null
+++ b/commons/nifi-utils/src/test/resources/logback-test.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="30 seconds">
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+ <logger name="org.apache.nifi" level="DEBUG"/>
+
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ </root>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/pom.xml
----------------------------------------------------------------------
diff --git a/commons/pom.xml b/commons/pom.xml
index fc7a1c3..19a7ae0 100644
--- a/commons/pom.xml
+++ b/commons/pom.xml
@@ -29,22 +29,16 @@
<name>NiFi :: Commons Parent</name>
<modules>
- <module>core-flowfile-attributes</module>
<module>data-provenance-utils</module>
<module>flowfile-packager</module>
- <module>naive-search-ring-buffer</module>
<module>nifi-expression-language</module>
- <module>nifi-file-utils</module>
<module>nifi-logging-utils</module>
<module>nifi-properties</module>
<module>nifi-security-utils</module>
<module>nifi-socket-utils</module>
- <module>nifi-stream-utils</module>
<module>nifi-utils</module>
<module>nifi-web-utils</module>
<module>processor-utilities</module>
- <module>remote-communications-utils</module>
- <module>search-utils</module>
<module>wali</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/pom.xml b/commons/remote-communications-utils/pom.xml
deleted file mode 100644
index 4eb0133..0000000
--- a/commons/remote-communications-utils/pom.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>remote-communications-utils</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>remote-communications-utils</name>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
deleted file mode 100644
index 77c34c9..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-public class StandardVersionNegotiator implements VersionNegotiator {
-
- private final List<Integer> versions;
- private int curVersion;
-
- public StandardVersionNegotiator(final int... supportedVersions) {
- if (Objects.requireNonNull(supportedVersions).length == 0) {
- throw new IllegalArgumentException("At least one version must be supported");
- }
-
- final List<Integer> supported = new ArrayList<>();
- for (final int version : supportedVersions) {
- supported.add(version);
- }
- this.versions = Collections.unmodifiableList(supported);
- this.curVersion = supportedVersions[0];
- }
-
- @Override
- public int getVersion() {
- return curVersion;
- }
-
- @Override
- public void setVersion(final int version) throws IllegalArgumentException {
- if (!isVersionSupported(version)) {
- throw new IllegalArgumentException("Version " + version + " is not supported");
- }
-
- this.curVersion = version;
- }
-
- @Override
- public int getPreferredVersion() {
- return versions.get(0);
- }
-
- @Override
- public Integer getPreferredVersion(final int maxVersion) {
- for (final Integer version : this.versions) {
- if (maxVersion >= version) {
- return version;
- }
- }
- return null;
- }
-
- @Override
- public boolean isVersionSupported(final int version) {
- return versions.contains(version);
- }
-
- @Override
- public List<Integer> getSupportedVersions() {
- return versions;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
deleted file mode 100644
index 74f9b3d..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote;
-
-import java.util.List;
-
-public interface VersionNegotiator {
-
- /**
- * @return the currently configured Version of this resource
- */
- int getVersion();
-
- /**
- * Sets the version of this resource to the specified version. Only the
- * lower byte of the version is relevant.
- *
- * @param version
- * @throws IllegalArgumentException if the given Version is not supported by
- * this resource, as is indicated by the {@link #isVersionSupported(int)}
- * method
- */
- void setVersion(int version) throws IllegalArgumentException;
-
- /**
- *
- * @return the Version of this resource that is preferred
- */
- int getPreferredVersion();
-
- /**
- * Gets the preferred version of this resource that is no greater than the
- * given maxVersion. If no acceptable version exists that is less than
- * <code>maxVersion</code>, then <code>null</code> is returned
- *
- * @param maxVersion
- * @return
- */
- Integer getPreferredVersion(int maxVersion);
-
- /**
- * Indicates whether or not the specified version is supported by this
- * resource
- *
- * @param version
- * @return
- */
- boolean isVersionSupported(int version);
-
- List<Integer> getSupportedVersions();
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
deleted file mode 100644
index 05fd915..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.exception;
-
-/**
- * Indicates that the user disabled transmission while communications were
- * taking place with a peer
- */
-public class TransmissionDisabledException extends RuntimeException {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
deleted file mode 100644
index 71cf894..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.zip.DataFormatException;
-import java.util.zip.Inflater;
-
-public class CompressionInputStream extends InputStream {
-
- private final InputStream in;
- private final Inflater inflater;
-
- private byte[] compressedBuffer;
- private byte[] buffer;
-
- private int bufferIndex;
- private boolean eos = false; // whether or not we've reached the end of stream
- private boolean allDataRead = false; // different from eos b/c eos means allDataRead == true && buffer is empty
-
- private final byte[] fourByteBuffer = new byte[4];
-
- public CompressionInputStream(final InputStream in) {
- this.in = in;
- inflater = new Inflater();
-
- buffer = new byte[0];
- compressedBuffer = new byte[0];
- bufferIndex = 1;
- }
-
- private String toHex(final byte[] array) {
- final StringBuilder sb = new StringBuilder("0x");
- for (final byte b : array) {
- final String hex = Integer.toHexString(b).toUpperCase();
- if (hex.length() == 1) {
- sb.append("0");
- }
- sb.append(hex);
- }
- return sb.toString();
- }
-
- protected void readChunkHeader() throws IOException {
- // Ensure that we have a valid SYNC chunk
- fillBuffer(fourByteBuffer);
- if (!Arrays.equals(CompressionOutputStream.SYNC_BYTES, fourByteBuffer)) {
- throw new IOException("Invalid CompressionInputStream. Expected first 4 bytes to be 'SYNC' but were " + toHex(fourByteBuffer));
- }
-
- // determine the size of the decompressed buffer
- fillBuffer(fourByteBuffer);
- buffer = new byte[toInt(fourByteBuffer)];
-
- // determine the size of the compressed buffer
- fillBuffer(fourByteBuffer);
- compressedBuffer = new byte[toInt(fourByteBuffer)];
-
- bufferIndex = buffer.length; // indicate that buffer is empty
- }
-
- private int toInt(final byte[] data) {
- return ((data[0] & 0xFF) << 24)
- | ((data[1] & 0xFF) << 16)
- | ((data[2] & 0xFF) << 8)
- | (data[3] & 0xFF);
- }
-
- protected void bufferAndDecompress() throws IOException {
- if (allDataRead) {
- eos = true;
- return;
- }
-
- readChunkHeader();
- fillBuffer(compressedBuffer);
-
- inflater.setInput(compressedBuffer);
- try {
- inflater.inflate(buffer);
- } catch (final DataFormatException e) {
- throw new IOException(e);
- }
- inflater.reset();
-
- bufferIndex = 0;
- final int moreDataByte = in.read();
- if (moreDataByte < 1) {
- allDataRead = true;
- } else if (moreDataByte > 1) {
- throw new IOException("Expected indicator of whether or not more data was to come (-1, 0, or 1) but got " + moreDataByte);
- }
- }
-
- private void fillBuffer(final byte[] buffer) throws IOException {
- int len;
- int bytesLeft = buffer.length;
- int bytesRead = 0;
- while (bytesLeft > 0 && (len = in.read(buffer, bytesRead, bytesLeft)) > 0) {
- bytesLeft -= len;
- bytesRead += len;
- }
-
- if (bytesRead < buffer.length) {
- throw new EOFException();
- }
- }
-
- private boolean isBufferEmpty() {
- return bufferIndex >= buffer.length;
- }
-
- @Override
- public int read() throws IOException {
- if (eos) {
- return -1;
- }
-
- if (isBufferEmpty()) {
- bufferAndDecompress();
- }
-
- if (isBufferEmpty()) {
- eos = true;
- return -1;
- }
-
- return buffer[bufferIndex++];
- }
-
- @Override
- public int read(final byte[] b) throws IOException {
- return read(b, 0, b.length);
- }
-
- @Override
- public int read(final byte[] b, final int off, final int len) throws IOException {
- if (eos) {
- return -1;
- }
-
- if (isBufferEmpty()) {
- bufferAndDecompress();
- }
-
- if (isBufferEmpty()) {
- eos = true;
- return -1;
- }
-
- final int free = buffer.length - bufferIndex;
- final int bytesToTransfer = Math.min(len, free);
- System.arraycopy(buffer, bufferIndex, b, off, bytesToTransfer);
- bufferIndex += bytesToTransfer;
-
- return bytesToTransfer;
- }
-
- /**
- * Does nothing. Does NOT close underlying InputStream
- * @throws java.io.IOException
- */
- @Override
- public void close() throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
deleted file mode 100644
index bc46b0f..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.zip.Deflater;
-
-public class CompressionOutputStream extends OutputStream {
-
- public static final byte[] SYNC_BYTES = new byte[]{'S', 'Y', 'N', 'C'};
-
- public static final int DEFAULT_COMPRESSION_LEVEL = 1;
- public static final int DEFAULT_BUFFER_SIZE = 64 << 10;
- public static final int MIN_BUFFER_SIZE = 8 << 10;
-
- private final OutputStream out;
- private final Deflater deflater;
-
- private final byte[] buffer;
- private final byte[] compressed;
-
- private int bufferIndex = 0;
- private boolean dataWritten = false;
-
- public CompressionOutputStream(final OutputStream outStream) {
- this(outStream, DEFAULT_BUFFER_SIZE);
- }
-
- public CompressionOutputStream(final OutputStream outStream, final int bufferSize) {
- this(outStream, bufferSize, DEFAULT_COMPRESSION_LEVEL, Deflater.DEFAULT_STRATEGY);
- }
-
- public CompressionOutputStream(final OutputStream outStream, final int bufferSize, final int level, final int strategy) {
- if (bufferSize < MIN_BUFFER_SIZE) {
- throw new IllegalArgumentException("Buffer size must be at least " + MIN_BUFFER_SIZE);
- }
-
- this.out = outStream;
- this.deflater = new Deflater(level);
- this.deflater.setStrategy(strategy);
- buffer = new byte[bufferSize];
- compressed = new byte[bufferSize + 64];
- }
-
- /**
- * Compresses the currently buffered chunk of data and sends it to the
- * output stream
- *
- * @throws IOException
- */
- protected void compressAndWrite() throws IOException {
- if (bufferIndex <= 0) {
- return;
- }
-
- deflater.setInput(buffer, 0, bufferIndex);
- deflater.finish();
- final int compressedBytes = deflater.deflate(compressed);
-
- writeChunkHeader(compressedBytes);
- out.write(compressed, 0, compressedBytes);
-
- bufferIndex = 0;
- deflater.reset();
- }
-
- private void writeChunkHeader(final int compressedBytes) throws IOException {
- // If we have already written data, write out a '1' to indicate that we have more data; when we close
- // the stream, we instead write a '0' to indicate that we are finished sending data.
- if (dataWritten) {
- out.write(1);
- }
- out.write(SYNC_BYTES);
- dataWritten = true;
-
- writeInt(out, bufferIndex);
- writeInt(out, compressedBytes);
- }
-
- private void writeInt(final OutputStream out, final int val) throws IOException {
- out.write(val >>> 24);
- out.write(val >>> 16);
- out.write(val >>> 8);
- out.write(val);
- }
-
- protected boolean bufferFull() {
- return bufferIndex >= buffer.length;
- }
-
- @Override
- public void write(final int b) throws IOException {
- buffer[bufferIndex++] = (byte) (b & 0xFF);
- if (bufferFull()) {
- compressAndWrite();
- }
- }
-
- @Override
- public void write(final byte[] b) throws IOException {
- write(b, 0, b.length);
- }
-
- @Override
- public void write(final byte[] b, final int off, final int len) throws IOException {
- int bytesLeft = len;
- while (bytesLeft > 0) {
- final int free = buffer.length - bufferIndex;
- final int bytesThisIteration = Math.min(bytesLeft, free);
- System.arraycopy(b, off + len - bytesLeft, buffer, bufferIndex, bytesThisIteration);
- bufferIndex += bytesThisIteration;
-
- bytesLeft -= bytesThisIteration;
- if (bufferFull()) {
- compressAndWrite();
- }
- }
- }
-
- @Override
- public void flush() throws IOException {
- compressAndWrite();
- super.flush();
- }
-
- @Override
- public void close() throws IOException {
- compressAndWrite();
- out.write(0); // indicate that the stream is finished.
- out.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
deleted file mode 100644
index e03dfbf..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.nifi.remote.exception.TransmissionDisabledException;
-
-public class InterruptableInputStream extends InputStream {
-
- private volatile boolean interrupted = false;
- private final InputStream in;
-
- public InterruptableInputStream(final InputStream in) {
- this.in = in;
- }
-
- @Override
- public int read() throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- return in.read();
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- return in.read(b);
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- return in.read(b, off, len);
- }
-
- @Override
- public int available() throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- return in.available();
- }
-
- @Override
- public void close() throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- in.close();
- }
-
- @Override
- public synchronized void mark(int readlimit) {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- in.mark(readlimit);
- }
-
- @Override
- public boolean markSupported() {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- return in.markSupported();
- }
-
- @Override
- public synchronized void reset() throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- in.reset();
- }
-
- @Override
- public long skip(long n) throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- return in.skip(n);
- }
-
- public void interrupt() {
- interrupted = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
deleted file mode 100644
index cba5be6..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.nifi.remote.exception.TransmissionDisabledException;
-
-public class InterruptableOutputStream extends OutputStream {
-
- private final OutputStream out;
- private volatile boolean interrupted = false;
-
- public InterruptableOutputStream(final OutputStream out) {
- this.out = out;
- }
-
- @Override
- public void write(int b) throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- out.write(b);
- }
-
- @Override
- public void write(byte[] b) throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- out.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- out.write(b, off, len);
- }
-
- @Override
- public void close() throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- out.close();
- }
-
- @Override
- public void flush() throws IOException {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- out.flush();
- }
-
- public void interrupt() {
- this.interrupted = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
deleted file mode 100644
index 68913bd..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io.socket;
-
-import java.nio.ByteBuffer;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class BufferStateManager {
-
- private static final Logger logger = LoggerFactory.getLogger(BufferStateManager.class);
-
- private ByteBuffer buffer;
- private Direction direction = Direction.WRITE;
-
- public BufferStateManager(final ByteBuffer buffer) {
- this.buffer = buffer;
- }
-
- public BufferStateManager(final ByteBuffer buffer, final Direction direction) {
- this.buffer = buffer;
- this.direction = direction;
- }
-
- /**
- * Ensures that the buffer is at least as big as the size specified,
- * resizing the buffer if necessary. This operation MAY change the direction
- * of the buffer.
- *
- * @param requiredSize
- */
- public void ensureSize(final int requiredSize) {
- if (buffer.capacity() < requiredSize) {
- final ByteBuffer newBuffer = ByteBuffer.allocate(requiredSize);
-
- // we have to read buffer so make sure the direction is correct.
- if (direction == Direction.WRITE) {
- buffer.flip();
- }
-
- // Copy from buffer to newBuffer
- newBuffer.put(buffer);
-
- // Swap the buffers
- buffer = newBuffer;
-
- // the new buffer is ready to be written to
- direction = Direction.WRITE;
- }
- }
-
- public ByteBuffer prepareForWrite(final int requiredSize) {
- ensureSize(requiredSize);
-
- if (direction == Direction.READ) {
- direction = Direction.WRITE;
- buffer.position(buffer.limit());
- }
-
- buffer.limit(buffer.capacity());
- return buffer;
- }
-
- public ByteBuffer prepareForRead(final int requiredSize) {
- ensureSize(requiredSize);
-
- if (direction == Direction.WRITE) {
- direction = Direction.READ;
- buffer.flip();
- }
-
- return buffer;
- }
-
- /**
- * Clears the contents of the buffer and sets direction to WRITE
- */
- public void clear() {
- logger.debug("Clearing {}", buffer);
- buffer.clear();
- direction = Direction.WRITE;
- }
-
- public void compact() {
- final String before = buffer.toString();
- buffer.compact();
- logger.debug("Before compact: {}, after: {}", before, buffer);
- direction = Direction.WRITE;
- }
-
- public static enum Direction {
-
- READ, WRITE;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
deleted file mode 100644
index 32a3f26..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io.socket;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.SocketTimeoutException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedByInterruptException;
-import java.nio.channels.SocketChannel;
-import java.util.concurrent.TimeUnit;
-
-public class SocketChannelInputStream extends InputStream {
-
- private static final long CHANNEL_EMPTY_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
- private final SocketChannel channel;
- private volatile int timeoutMillis = 30000;
-
- private final ByteBuffer oneByteBuffer = ByteBuffer.allocate(1);
- private Byte bufferedByte = null;
-
- public SocketChannelInputStream(final SocketChannel socketChannel) throws IOException {
- // this class expects a non-blocking channel
- socketChannel.configureBlocking(false);
- this.channel = socketChannel;
- }
-
- public void setTimeout(final int timeoutMillis) {
- this.timeoutMillis = timeoutMillis;
- }
-
- @Override
- public int read() throws IOException {
- if (bufferedByte != null) {
- final int retVal = bufferedByte & 0xFF;
- bufferedByte = null;
- return retVal;
- }
-
- oneByteBuffer.flip();
- oneByteBuffer.clear();
-
- final long maxTime = System.currentTimeMillis() + timeoutMillis;
- int bytesRead;
- do {
- bytesRead = channel.read(oneByteBuffer);
- if (bytesRead == 0) {
- if (System.currentTimeMillis() > maxTime) {
- throw new SocketTimeoutException("Timed out reading from socket");
- }
- try {
- TimeUnit.NANOSECONDS.sleep(CHANNEL_EMPTY_WAIT_NANOS);
- } catch (InterruptedException e) {
- close();
- Thread.currentThread().interrupt(); // set the interrupt status
- throw new ClosedByInterruptException(); // simulate an interrupted blocked read operation
- }
- }
- } while (bytesRead == 0);
-
- if (bytesRead == -1) {
- return -1;
- }
- oneByteBuffer.flip();
- return oneByteBuffer.get() & 0xFF;
- }
-
- @Override
- public int read(final byte[] b) throws IOException {
- return read(b, 0, b.length);
- }
-
- @Override
- public int read(final byte[] b, final int off, final int len) throws IOException {
- if (bufferedByte != null) {
- final byte retVal = bufferedByte;
- bufferedByte = null;
- b[off] = retVal;
- return 1;
- }
-
- final ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
-
- final long maxTime = System.currentTimeMillis() + timeoutMillis;
- int bytesRead;
- do {
- bytesRead = channel.read(buffer);
- if (bytesRead == 0) {
- if (System.currentTimeMillis() > maxTime) {
- throw new SocketTimeoutException("Timed out reading from socket");
- }
- try {
- TimeUnit.NANOSECONDS.sleep(CHANNEL_EMPTY_WAIT_NANOS);
- } catch (InterruptedException e) {
- close();
- Thread.currentThread().interrupt(); // set the interrupt status
- throw new ClosedByInterruptException(); // simulate an interrupted blocked read operation
- }
- }
- } while (bytesRead == 0);
-
- return bytesRead;
- }
-
- @Override
- public int available() throws IOException {
- if (bufferedByte != null) {
- return 1;
- }
-
- isDataAvailable(); // attempt to read from socket
- return (bufferedByte == null) ? 0 : 1;
- }
-
- public boolean isDataAvailable() throws IOException {
- if (bufferedByte != null) {
- return true;
- }
-
- oneByteBuffer.flip();
- oneByteBuffer.clear();
- final int bytesRead = channel.read(oneByteBuffer);
- if (bytesRead == -1) {
- throw new EOFException("Peer has closed the stream");
- }
- if (bytesRead > 0) {
- oneByteBuffer.flip();
- bufferedByte = oneByteBuffer.get();
- return true;
- }
- return false;
- }
-
- /**
- * Closes the underlying socket channel.
- * @throws java.io.IOException
- */
- @Override
- public void close() throws IOException {
- channel.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
deleted file mode 100644
index 77049ad..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io.socket;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.SocketTimeoutException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedByInterruptException;
-import java.nio.channels.SocketChannel;
-import java.util.concurrent.TimeUnit;
-
-public class SocketChannelOutputStream extends OutputStream {
-
- private static final long CHANNEL_FULL_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
- private final SocketChannel channel;
- private volatile int timeout = 30000;
-
- private final ByteBuffer oneByteBuffer = ByteBuffer.allocate(1);
-
- public SocketChannelOutputStream(final SocketChannel socketChannel) throws IOException {
- // this class expects a non-blocking channel
- socketChannel.configureBlocking(false);
- this.channel = socketChannel;
- }
-
- public void setTimeout(final int timeoutMillis) {
- this.timeout = timeoutMillis;
- }
-
- @Override
- public void write(final int b) throws IOException {
- oneByteBuffer.flip();
- oneByteBuffer.clear();
- oneByteBuffer.put((byte) b);
- oneByteBuffer.flip();
-
- final int timeoutMillis = this.timeout;
- long maxTime = System.currentTimeMillis() + timeoutMillis;
- int bytesWritten;
- while (oneByteBuffer.hasRemaining()) {
- bytesWritten = channel.write(oneByteBuffer);
- if (bytesWritten == 0) {
- if (System.currentTimeMillis() > maxTime) {
- throw new SocketTimeoutException("Timed out writing to socket");
- }
- try {
- TimeUnit.NANOSECONDS.sleep(CHANNEL_FULL_WAIT_NANOS);
- } catch (InterruptedException e) {
- close();
- Thread.currentThread().interrupt(); // set the interrupt status
- throw new ClosedByInterruptException(); // simulate an interrupted blocked write operation
- }
- } else {
- return;
- }
- }
- }
-
- @Override
- public void write(final byte[] b) throws IOException {
- write(b, 0, b.length);
- }
-
- @Override
- public void write(final byte[] b, final int off, final int len) throws IOException {
- final ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
-
- final int timeoutMillis = this.timeout;
- long maxTime = System.currentTimeMillis() + timeoutMillis;
- int bytesWritten;
- while (buffer.hasRemaining()) {
- bytesWritten = channel.write(buffer);
- if (bytesWritten == 0) {
- if (System.currentTimeMillis() > maxTime) {
- throw new SocketTimeoutException("Timed out writing to socket");
- }
- try {
- TimeUnit.NANOSECONDS.sleep(CHANNEL_FULL_WAIT_NANOS);
- } catch (InterruptedException e) {
- close();
- Thread.currentThread().interrupt(); // set the interrupt status
- throw new ClosedByInterruptException(); // simulate an interrupted blocked write operation
- }
- } else {
- maxTime = System.currentTimeMillis() + timeoutMillis;
- }
- }
- }
-
- /**
- * Closes the underlying SocketChannel
- * @throws java.io.IOException
- */
- @Override
- public void close() throws IOException {
- channel.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
deleted file mode 100644
index 5810488..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
+++ /dev/null
@@ -1,602 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io.socket.ssl;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketAddress;
-import java.net.SocketTimeoutException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedByInterruptException;
-import java.nio.channels.SocketChannel;
-import java.util.concurrent.TimeUnit;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLEngineResult;
-import javax.net.ssl.SSLEngineResult.Status;
-import javax.net.ssl.SSLHandshakeException;
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.security.cert.CertificateExpiredException;
-import javax.security.cert.CertificateNotYetValidException;
-import javax.security.cert.X509Certificate;
-
-import org.apache.nifi.remote.exception.TransmissionDisabledException;
-import org.apache.nifi.remote.io.socket.BufferStateManager;
-import org.apache.nifi.remote.io.socket.BufferStateManager.Direction;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SSLSocketChannel implements Closeable {
-
- public static final int MAX_WRITE_SIZE = 65536;
-
- private static final Logger logger = LoggerFactory.getLogger(SSLSocketChannel.class);
- private static final long BUFFER_FULL_EMPTY_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
-
- private final String hostname;
- private final int port;
- private final SSLEngine engine;
- private final SocketAddress socketAddress;
-
- private BufferStateManager streamInManager;
- private BufferStateManager streamOutManager;
- private BufferStateManager appDataManager;
-
- private SocketChannel channel;
-
- private final byte[] oneByteBuffer = new byte[1];
-
- private int timeoutMillis = 30000;
- private volatile boolean connected = false;
- private boolean handshaking = false;
- private boolean closed = false;
- private volatile boolean interrupted = false;
-
- public SSLSocketChannel(final SSLContext sslContext, final String hostname, final int port, final boolean client) throws IOException {
- this.socketAddress = new InetSocketAddress(hostname, port);
- this.channel = SocketChannel.open();
- this.hostname = hostname;
- this.port = port;
- this.engine = sslContext.createSSLEngine();
- this.engine.setUseClientMode(client);
- engine.setNeedClientAuth(true);
-
- streamInManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
- streamOutManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
- appDataManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getApplicationBufferSize()));
- }
-
- public SSLSocketChannel(final SSLContext sslContext, final SocketChannel socketChannel, final boolean client) throws IOException {
- if (!socketChannel.isConnected()) {
- throw new IllegalArgumentException("Cannot pass an un-connected SocketChannel");
- }
-
- this.channel = socketChannel;
-
- this.socketAddress = socketChannel.getRemoteAddress();
- final Socket socket = socketChannel.socket();
- this.hostname = socket.getInetAddress().getHostName();
- this.port = socket.getPort();
-
- this.engine = sslContext.createSSLEngine();
- this.engine.setUseClientMode(client);
- engine.setNeedClientAuth(true);
-
- streamInManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
- streamOutManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
- appDataManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getApplicationBufferSize()));
- }
-
- public void setTimeout(final int millis) {
- this.timeoutMillis = millis;
- }
-
- public int getTimeout() {
- return timeoutMillis;
- }
-
- public void connect() throws SSLHandshakeException, IOException {
- try {
- channel.configureBlocking(false);
- if (!channel.isConnected()) {
- final long startTime = System.currentTimeMillis();
-
- if (!channel.connect(socketAddress)) {
- while (!channel.finishConnect()) {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
- if (System.currentTimeMillis() > startTime + timeoutMillis) {
- throw new SocketTimeoutException("Timed out connecting to " + hostname + ":" + port);
- }
-
- try {
- Thread.sleep(50L);
- } catch (final InterruptedException e) {
- }
- }
- }
- }
- engine.beginHandshake();
-
- performHandshake();
- logger.debug("{} Successfully completed SSL handshake", this);
-
- streamInManager.clear();
- streamOutManager.clear();
- appDataManager.clear();
-
- connected = true;
- } catch (final Exception e) {
- logger.error("{} Failed to connect due to {}", this, e);
- if (logger.isDebugEnabled()) {
- logger.error("", e);
- }
- closeQuietly(channel);
- engine.closeInbound();
- engine.closeOutbound();
- throw e;
- }
- }
-
- public String getDn() throws CertificateExpiredException, CertificateNotYetValidException, SSLPeerUnverifiedException {
- final X509Certificate[] certs = engine.getSession().getPeerCertificateChain();
- if (certs == null || certs.length == 0) {
- throw new SSLPeerUnverifiedException("No certificates found");
- }
-
- final X509Certificate cert = certs[0];
- cert.checkValidity();
- return cert.getSubjectDN().getName().trim();
- }
-
- private void performHandshake() throws IOException {
- // Generate handshake message
- final byte[] emptyMessage = new byte[0];
- handshaking = true;
- logger.debug("{} Performing Handshake", this);
-
- try {
- while (true) {
- switch (engine.getHandshakeStatus()) {
- case FINISHED:
- return;
- case NEED_WRAP: {
- final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);
-
- final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
-
- final SSLEngineResult wrapHelloResult = engine.wrap(appDataOut, outboundBuffer);
- if (wrapHelloResult.getStatus() == Status.BUFFER_OVERFLOW) {
- streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
- continue;
- }
-
- if (wrapHelloResult.getStatus() != Status.OK) {
- throw new SSLHandshakeException("Could not generate SSL Handshake information: SSLEngineResult: "
- + wrapHelloResult.toString());
- }
-
- logger.trace("{} Handshake response after wrapping: {}", this, wrapHelloResult);
-
- final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
- final int bytesToSend = readableStreamOut.remaining();
- writeFully(readableStreamOut);
- logger.trace("{} Sent {} bytes of wrapped data for handshake", this, bytesToSend);
-
- streamOutManager.clear();
- }
- continue;
- case NEED_UNWRAP: {
- final ByteBuffer readableDataIn = streamInManager.prepareForRead(0);
- final ByteBuffer appData = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
-
- // Read handshake response from other side
- logger.trace("{} Unwrapping: {} to {}", new Object[]{this, readableDataIn, appData});
- SSLEngineResult handshakeResponseResult = engine.unwrap(readableDataIn, appData);
- logger.trace("{} Handshake response after unwrapping: {}", this, handshakeResponseResult);
-
- if (handshakeResponseResult.getStatus() == Status.BUFFER_UNDERFLOW) {
- final ByteBuffer writableDataIn = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
- final int bytesRead = readData(writableDataIn);
- if (bytesRead > 0) {
- logger.trace("{} Read {} bytes for handshake", this, bytesRead);
- }
-
- if (bytesRead < 0) {
- throw new SSLHandshakeException("Reached End-of-File marker while performing handshake");
- }
- } else if (handshakeResponseResult.getStatus() == Status.CLOSED) {
- throw new IOException("Channel was closed by peer during handshake");
- } else {
- streamInManager.compact();
- appDataManager.clear();
- }
- }
- break;
- case NEED_TASK:
- performTasks();
- continue;
- case NOT_HANDSHAKING:
- return;
- }
- }
- } finally {
- handshaking = false;
- }
- }
-
- private void performTasks() {
- Runnable runnable;
- while ((runnable = engine.getDelegatedTask()) != null) {
- runnable.run();
- }
- }
-
- private void closeQuietly(final Closeable closeable) {
- try {
- closeable.close();
- } catch (final Exception e) {
- }
- }
-
- private int readData(final ByteBuffer dest) throws IOException {
- final long startTime = System.currentTimeMillis();
-
- while (true) {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- if (dest.remaining() == 0) {
- return 0;
- }
-
- final int readCount = channel.read(dest);
-
- if (readCount == 0) {
- if (System.currentTimeMillis() > startTime + timeoutMillis) {
- throw new SocketTimeoutException("Timed out reading from socket connected to " + hostname + ":" + port);
- }
- try {
- TimeUnit.NANOSECONDS.sleep(BUFFER_FULL_EMPTY_WAIT_NANOS);
- } catch (InterruptedException e) {
- close();
- Thread.currentThread().interrupt(); // set the interrupt status
- throw new ClosedByInterruptException();
- }
-
- continue;
- }
-
- logger.trace("{} Read {} bytes", this, readCount);
- return readCount;
- }
- }
-
- private Status encryptAndWriteFully(final BufferStateManager src) throws IOException {
- SSLEngineResult result = null;
-
- final ByteBuffer buff = src.prepareForRead(0);
- final ByteBuffer outBuff = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
-
- logger.trace("{} Encrypting {} bytes", this, buff.remaining());
- while (buff.remaining() > 0) {
- result = engine.wrap(buff, outBuff);
- if (result.getStatus() == Status.OK) {
- final ByteBuffer readableOutBuff = streamOutManager.prepareForRead(0);
- writeFully(readableOutBuff);
- streamOutManager.clear();
- } else {
- return result.getStatus();
- }
- }
-
- return result.getStatus();
- }
-
- private void writeFully(final ByteBuffer src) throws IOException {
- long lastByteWrittenTime = System.currentTimeMillis();
-
- int bytesWritten = 0;
- while (src.hasRemaining()) {
- if (interrupted) {
- throw new TransmissionDisabledException();
- }
-
- final int written = channel.write(src);
- bytesWritten += written;
- final long now = System.currentTimeMillis();
- if (written > 0) {
- lastByteWrittenTime = now;
- } else {
- if (now > lastByteWrittenTime + timeoutMillis) {
- throw new SocketTimeoutException("Timed out writing to socket connected to " + hostname + ":" + port);
- }
- try {
- TimeUnit.NANOSECONDS.sleep(BUFFER_FULL_EMPTY_WAIT_NANOS);
- } catch (final InterruptedException e) {
- close();
- Thread.currentThread().interrupt(); // set the interrupt status
- throw new ClosedByInterruptException();
- }
- }
- }
-
- logger.trace("{} Wrote {} bytes", this, bytesWritten);
- }
-
- public boolean isClosed() {
- if (closed) {
- return true;
- }
- // need to detect if peer has sent closure handshake...if so the answer is true
- final ByteBuffer writableInBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
- int readCount = 0;
- try {
- readCount = channel.read(writableInBuffer);
- } catch (IOException e) {
- logger.error("{} Failed to readData due to {}", new Object[]{this, e});
- if (logger.isDebugEnabled()) {
- logger.error("", e);
- }
- readCount = -1; // treat the condition same as if End of Stream
- }
- if (readCount == 0) {
- return false;
- }
- if (readCount > 0) {
- logger.trace("{} Read {} bytes", this, readCount);
-
- final ByteBuffer streamInBuffer = streamInManager.prepareForRead(1);
- final ByteBuffer appDataBuffer = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
- try {
- SSLEngineResult unwrapResponse = engine.unwrap(streamInBuffer, appDataBuffer);
- logger.trace("{} When checking if closed, (handshake={}) Unwrap response: {}", new Object[]{this, handshaking, unwrapResponse});
- if (unwrapResponse.getStatus().equals(Status.CLOSED)) {
- // Drain the incoming TCP buffer
- final ByteBuffer discardBuffer = ByteBuffer.allocate(8192);
- int bytesDiscarded = channel.read(discardBuffer);
- while (bytesDiscarded > 0) {
- discardBuffer.clear();
- bytesDiscarded = channel.read(discardBuffer);
- }
- engine.closeInbound();
- } else {
- streamInManager.compact();
- return false;
- }
- } catch (IOException e) {
- logger.error("{} Failed to check if closed due to {}. Closing channel.", new Object[]{this, e});
- if (logger.isDebugEnabled()) {
- logger.error("", e);
- }
- }
- }
- // either readCount is -1, indicating an end of stream, or the peer sent a closure handshake
- // so go ahead and close down the channel
- closeQuietly(channel.socket());
- closeQuietly(channel);
- closed = true;
- return true;
- }
-
- @Override
- public void close() throws IOException {
- logger.debug("{} Closing Connection", this);
- if (channel == null) {
- return;
- }
-
- if (closed) {
- return;
- }
-
- try {
- engine.closeOutbound();
-
- final byte[] emptyMessage = new byte[0];
-
- final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);
- final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
- final SSLEngineResult handshakeResult = engine.wrap(appDataOut, outboundBuffer);
-
- if (handshakeResult.getStatus() != Status.CLOSED) {
- throw new IOException("Invalid close state - will not send network data");
- }
-
- final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
- writeFully(readableStreamOut);
- } finally {
- // Drain the incoming TCP buffer
- final ByteBuffer discardBuffer = ByteBuffer.allocate(8192);
- try {
- int bytesDiscarded = channel.read(discardBuffer);
- while (bytesDiscarded > 0) {
- discardBuffer.clear();
- bytesDiscarded = channel.read(discardBuffer);
- }
- } catch (Exception e) {
- }
-
- closeQuietly(channel.socket());
- closeQuietly(channel);
- closed = true;
- }
- }
-
- private int copyFromAppDataBuffer(final byte[] buffer, final int offset, final int len) {
- // If any data already exists in the application data buffer, copy it to the buffer.
- final ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
-
- final int appDataRemaining = appDataBuffer.remaining();
- if (appDataRemaining > 0) {
- final int bytesToCopy = Math.min(len, appDataBuffer.remaining());
- appDataBuffer.get(buffer, offset, bytesToCopy);
-
- final int bytesCopied = appDataRemaining - appDataBuffer.remaining();
- logger.trace("{} Copied {} ({}) bytes from unencrypted application buffer to user space",
- new Object[]{this, bytesToCopy, bytesCopied});
- return bytesCopied;
- }
- return 0;
- }
-
- public int available() throws IOException {
- ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
- ByteBuffer streamDataBuffer = streamInManager.prepareForRead(1);
- final int buffered = appDataBuffer.remaining() + streamDataBuffer.remaining();
- if (buffered > 0) {
- return buffered;
- }
-
- final boolean wasAbleToRead = isDataAvailable();
- if (!wasAbleToRead) {
- return 0;
- }
-
- appDataBuffer = appDataManager.prepareForRead(1);
- streamDataBuffer = streamInManager.prepareForRead(1);
- return appDataBuffer.remaining() + streamDataBuffer.remaining();
- }
-
- public boolean isDataAvailable() throws IOException {
- final ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
- final ByteBuffer streamDataBuffer = streamInManager.prepareForRead(1);
-
- if (appDataBuffer.remaining() > 0 || streamDataBuffer.remaining() > 0) {
- return true;
- }
-
- final ByteBuffer writableBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
- final int bytesRead = channel.read(writableBuffer);
- return (bytesRead > 0);
- }
-
- public int read() throws IOException {
- final int bytesRead = read(oneByteBuffer);
- if (bytesRead == -1) {
- return -1;
- }
- return oneByteBuffer[0] & 0xFF;
- }
-
- public int read(final byte[] buffer) throws IOException {
- return read(buffer, 0, buffer.length);
- }
-
- public int read(final byte[] buffer, final int offset, final int len) throws IOException {
- logger.debug("{} Reading up to {} bytes of data", this, len);
-
- if (!connected) {
- connect();
- }
-
- int copied = copyFromAppDataBuffer(buffer, offset, len);
- if (copied > 0) {
- return copied;
- }
-
- appDataManager.clear();
-
- while (true) {
- // prepare buffers and call unwrap
- final ByteBuffer streamInBuffer = streamInManager.prepareForRead(1);
- SSLEngineResult unwrapResponse = null;
- final ByteBuffer appDataBuffer = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
- unwrapResponse = engine.unwrap(streamInBuffer, appDataBuffer);
- logger.trace("{} When reading data, (handshake={}) Unwrap response: {}", new Object[]{this, handshaking, unwrapResponse});
-
- switch (unwrapResponse.getStatus()) {
- case BUFFER_OVERFLOW:
- throw new SSLHandshakeException("Buffer Overflow, which is not allowed to happen from an unwrap");
- case BUFFER_UNDERFLOW: {
-// appDataManager.prepareForRead(engine.getSession().getApplicationBufferSize());
-
- final ByteBuffer writableInBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
- final int bytesRead = readData(writableInBuffer);
- if (bytesRead < 0) {
- return -1;
- }
-
- continue;
- }
- case CLOSED:
- throw new IOException("Channel is closed");
- case OK: {
- copied = copyFromAppDataBuffer(buffer, offset, len);
- if (copied == 0) {
- throw new IOException("Failed to decrypt data");
- }
- streamInManager.compact();
- return copied;
- }
- }
- }
- }
-
- public void write(final int data) throws IOException {
- write(new byte[]{(byte) data}, 0, 1);
- }
-
- public void write(final byte[] data) throws IOException {
- write(data, 0, data.length);
- }
-
- public void write(final byte[] data, final int offset, final int len) throws IOException {
- logger.debug("{} Writing {} bytes of data", this, len);
-
- if (!connected) {
- connect();
- }
-
- int iterations = len / MAX_WRITE_SIZE;
- if (len % MAX_WRITE_SIZE > 0) {
- iterations++;
- }
-
- for (int i = 0; i < iterations; i++) {
- streamOutManager.clear();
- final int itrOffset = offset + i * MAX_WRITE_SIZE;
- final int itrLen = Math.min(len - itrOffset, MAX_WRITE_SIZE);
- final ByteBuffer byteBuffer = ByteBuffer.wrap(data, itrOffset, itrLen);
-
- final BufferStateManager buffMan = new BufferStateManager(byteBuffer, Direction.READ);
- final Status status = encryptAndWriteFully(buffMan);
- switch (status) {
- case BUFFER_OVERFLOW:
- streamOutManager.ensureSize(engine.getSession().getPacketBufferSize());
- appDataManager.ensureSize(engine.getSession().getApplicationBufferSize());
- continue;
- case OK:
- continue;
- case CLOSED:
- throw new IOException("Channel is closed");
- case BUFFER_UNDERFLOW:
- throw new AssertionError("Got Buffer Underflow but should not have...");
- }
- }
- }
-
- public void interrupt() {
- this.interrupted = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
deleted file mode 100644
index 154bd08..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io.socket.ssl;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-public class SSLSocketChannelInputStream extends InputStream {
-
- private final SSLSocketChannel channel;
-
- public SSLSocketChannelInputStream(final SSLSocketChannel channel) {
- this.channel = channel;
- }
-
- @Override
- public int read() throws IOException {
- return channel.read();
- }
-
- @Override
- public int read(final byte[] b) throws IOException {
- return channel.read(b);
- }
-
- @Override
- public int read(final byte[] b, final int off, final int len) throws IOException {
- return channel.read(b, off, len);
- }
-
- /**
- * Closes the underlying SSLSocketChannel, which will also close the
- * OutputStream and connection
- */
- @Override
- public void close() throws IOException {
- channel.close();
- }
-
- @Override
- public int available() throws IOException {
- return channel.available();
- }
-
- public boolean isDataAvailable() throws IOException {
- return available() > 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java b/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
deleted file mode 100644
index ce4e420..0000000
--- a/commons/remote-communications-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io.socket.ssl;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-public class SSLSocketChannelOutputStream extends OutputStream {
-
- private final SSLSocketChannel channel;
-
- public SSLSocketChannelOutputStream(final SSLSocketChannel channel) {
- this.channel = channel;
- }
-
- @Override
- public void write(final int b) throws IOException {
- channel.write(b);
- }
-
- @Override
- public void write(byte[] b) throws IOException {
- channel.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- channel.write(b, off, len);
- }
-
- /**
- * Closes the underlying SSLSocketChannel, which also will close the
- * InputStream and the connection
- */
- @Override
- public void close() throws IOException {
- channel.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/remote-communications-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
----------------------------------------------------------------------
diff --git a/commons/remote-communications-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java b/commons/remote-communications-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
deleted file mode 100644
index bd30a96..0000000
--- a/commons/remote-communications-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.remote.io;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-
-import org.apache.nifi.remote.io.CompressionInputStream;
-import org.apache.nifi.remote.io.CompressionOutputStream;
-
-import org.junit.Test;
-
-public class TestCompressionInputOutputStreams {
-
- @Test
- public void testSimple() throws IOException {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- final byte[] data = "Hello, World!".getBytes("UTF-8");
-
- final CompressionOutputStream cos = new CompressionOutputStream(baos);
- cos.write(data);
- cos.flush();
- cos.close();
-
- final byte[] compressedBytes = baos.toByteArray();
- final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
- final byte[] decompressed = readFully(cis);
-
- assertTrue(Arrays.equals(data, decompressed));
- }
-
- @Test
- public void testDataLargerThanBuffer() throws IOException {
- final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
-
- final StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 100; i++) {
- sb.append(str);
- }
- final byte[] data = sb.toString().getBytes("UTF-8");
-
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
- cos.write(data);
- cos.flush();
- cos.close();
-
- final byte[] compressedBytes = baos.toByteArray();
- final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
- final byte[] decompressed = readFully(cis);
-
- assertTrue(Arrays.equals(data, decompressed));
- }
-
- @Test
- public void testDataLargerThanBufferWhileFlushing() throws IOException {
- final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
- final byte[] data = str.getBytes("UTF-8");
-
- final StringBuilder sb = new StringBuilder();
- final byte[] data1024;
-
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
- for (int i = 0; i < 1024; i++) {
- cos.write(data);
- cos.flush();
- sb.append(str);
- }
- cos.close();
- data1024 = sb.toString().getBytes("UTF-8");
-
- final byte[] compressedBytes = baos.toByteArray();
- final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
- final byte[] decompressed = readFully(cis);
-
- assertTrue(Arrays.equals(data1024, decompressed));
- }
-
- @Test
- public void testSendingMultipleFilesBackToBackOnSameStream() throws IOException {
- final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
- final byte[] data = str.getBytes("UTF-8");
-
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
- for (int i = 0; i < 512; i++) {
- cos.write(data);
- cos.flush();
- }
- cos.close();
-
- final CompressionOutputStream cos2 = new CompressionOutputStream(baos, 8192);
- for (int i = 0; i < 512; i++) {
- cos2.write(data);
- cos2.flush();
- }
- cos2.close();
-
- final byte[] data512;
- final StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 512; i++) {
- sb.append(str);
- }
- data512 = sb.toString().getBytes("UTF-8");
-
- final byte[] compressedBytes = baos.toByteArray();
- final ByteArrayInputStream bais = new ByteArrayInputStream(compressedBytes);
-
- final CompressionInputStream cis = new CompressionInputStream(bais);
- final byte[] decompressed = readFully(cis);
- assertTrue(Arrays.equals(data512, decompressed));
-
- final CompressionInputStream cis2 = new CompressionInputStream(bais);
- final byte[] decompressed2 = readFully(cis2);
- assertTrue(Arrays.equals(data512, decompressed2));
- }
-
- private byte[] readFully(final InputStream in) throws IOException {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- final byte[] buffer = new byte[65536];
- int len;
- while ((len = in.read(buffer)) >= 0) {
- baos.write(buffer, 0, len);
- }
-
- return baos.toByteArray();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/search-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/search-utils/pom.xml b/commons/search-utils/pom.xml
deleted file mode 100644
index 7caacc8..0000000
--- a/commons/search-utils/pom.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>nifi-search-utils</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>search-utils</name>
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/search-utils/src/main/java/org/apache/nifi/util/search/Search.java
----------------------------------------------------------------------
diff --git a/commons/search-utils/src/main/java/org/apache/nifi/util/search/Search.java b/commons/search-utils/src/main/java/org/apache/nifi/util/search/Search.java
deleted file mode 100644
index 59b444a..0000000
--- a/commons/search-utils/src/main/java/org/apache/nifi/util/search/Search.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.search;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Set;
-
-import org.apache.nifi.util.search.ahocorasick.SearchState;
-
-/**
- * Defines an interface to search for content given a set of search terms. Any
- * implementation of search must be thread safe.
- *
- * @author
- * @param <T>
- */
-public interface Search<T> {
-
- /**
- * Establishes the dictionary of terms which will be searched in subsequent
- * search calls. This can be called only once
- *
- * @param terms
- */
- void initializeDictionary(Set<SearchTerm<T>> terms);
-
- /**
- * Searches the given input stream for matches between the already specified
- * dictionary and the contents scanned.
- *
- * @param haystack
- * @param findAll if true will find all matches if false will find only the
- * first match
- * @return SearchState containing results Map might be empty which indicates
- * no matches found but will not be null
- * @throws IOException Thrown for any exceptions occurring while searching.
- * @throws IllegalStateException if the dictionary has not yet been
- * initialized
- */
- SearchState<T> search(InputStream haystack, boolean findAll) throws IOException;
-
-}
[27/31] incubator-nifi git commit: NIFI-191 updated readme to reflect
export control and added readme to resources for distro
Posted by mc...@apache.org.
NIFI-191 updated readme to reflect export control and added readme to resources for distro
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/1da5a9c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/1da5a9c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/1da5a9c1
Branch: refs/heads/NIFI-182
Commit: 1da5a9c175134b822bac04813673a6536f2546e3
Parents: 965e0ab
Author: joewitt <jo...@apache.org>
Authored: Sat Dec 20 12:22:13 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sat Dec 20 12:22:13 2014 -0500
----------------------------------------------------------------------
README.md | 30 +++++++
.../resources/src/main/resources/README | 84 ++++++++++++++++++++
2 files changed, 114 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1da5a9c1/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 3f05134..bc4c008 100644
--- a/README.md
+++ b/README.md
@@ -63,3 +63,33 @@ have stabilized in a manner consistent with other successful ASF projects.
While incubation status is not necessarily a reflection of the completeness
or stability of the code, it does indicate that the project has yet to be
fully endorsed by the ASF.
+
+## Export Control
+
+This distribution includes cryptographic software. The country in which you
+currently reside may have restrictions on the import, possession, use, and/or
+re-export to another country, of encryption software. BEFORE using any
+encryption software, please check your country's laws, regulations and
+policies concerning the import, possession, or use, and re-export of encryption
+software, to see if this is permitted. See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and Security
+(BIS), has classified this software as Export Commodity Control Number (ECCN)
+5D002.C.1, which includes information security software using or performing
+cryptographic functions with asymmetric algorithms. The form and manner of this
+Apache Software Foundation distribution makes it eligible for export under the
+License Exception ENC Technology Software Unrestricted (TSU) exception (see the
+BIS Export Administration Regulations, Section 740.13) for both object code and
+source code.
+
+The following provides more details on the included cryptographic software:
+
+Apache NiFi uses BouncyCastle, Jasypt, JCraft Inc., and the built-in
+java cryptography libraries for SSL, SSH, and the protection
+of sensitive configuration parameters. See
+http://bouncycastle.org/about.html
+http://www.jasypt.org/faq.html
+http://jcraft.com/c-info.html
+http://www.oracle.com/us/products/export/export-regulations-345813.html
+for more details on each of these libraries cryptography features.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1da5a9c1/nar-bundles/framework-bundle/framework/resources/src/main/resources/README
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/README b/nar-bundles/framework-bundle/framework/resources/src/main/resources/README
new file mode 100644
index 0000000..97c4964
--- /dev/null
+++ b/nar-bundles/framework-bundle/framework/resources/src/main/resources/README
@@ -0,0 +1,84 @@
+***********************************************
+Introduction
+
+Apache NiFi is a dataflow system based on the concepts of flow-based programming. It is currently apart of the Apache Incubator.
+
+***********************************************
+Features
+
+Apache NiFi supports powerful and scalable directed graphs of data routing, transformation, and system mediation logic. Some of the high-level capabilities and objectives of Apache NiFi include:
+
+- Web-based user interface for seamless experience between design, control, feedback, and monitoring of data flows
+- Highly configurable along several dimensions of quality of service such as loss tolerant versus guaranteed delivery, low latency versus high throughput, and priority based queuing
+- Fine-grained data provenance for all data received, forked, joined, cloned, modified, sent, and ultimately dropped as data reaches its configured end-state
+- Component-based extension model along well defined interfaces enabling rapid development and effective testing
+
+************************************************
+Getting Started
+
+Execute <nifi install dir>/bin/nifi.sh start
+
+*************************************************
+Getting Help
+
+If you have questions, you can reach out to our mailing list: dev@nifi.incubator.apache.org
+([archive](http://mail-archives.apache.org/mod_mbox/incubator-nifi-dev)).
+We're also often available in IRC: #nifi on
+[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi).
+
+*************************************************
+Requirements
+
+JDK 1.7 or higher
+
+*************************************************
+License and Notices
+
+See
+ <nifi install dir>/LICENSE
+ <nifi install dir>/NOTICE
+
+**************************************************
+Disclaimer
+
+Apache NiFi is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+
+While incubation status is not necessarily a reflection of the completeness
+or stability of the code, it does indicate that the project has yet to be
+fully endorsed by the ASF.
+
+****************************************************
+Export Control
+
+This distribution includes cryptographic software. The country in which you
+currently reside may have restrictions on the import, possession, use, and/or
+re-export to another country, of encryption software. BEFORE using any
+encryption software, please check your country's laws, regulations and
+policies concerning the import, possession, or use, and re-export of encryption
+software, to see if this is permitted. See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and Security
+(BIS), has classified this software as Export Commodity Control Number (ECCN)
+5D002.C.1, which includes information security software using or performing
+cryptographic functions with asymmetric algorithms. The form and manner of this
+Apache Software Foundation distribution makes it eligible for export under the
+License Exception ENC Technology Software Unrestricted (TSU) exception (see the
+BIS Export Administration Regulations, Section 740.13) for both object code and
+source code.
+
+The following provides more details on the included cryptographic software:
+
+Apache NiFi uses BouncyCastle, Jasypt, JCraft Inc., and the built-in
+java cryptography libraries for SSL, SSH, and the protection
+of sensitive configuration parameters. See
+http://bouncycastle.org/about.html
+http://www.jasypt.org/faq.html
+http://jcraft.com/c-info.html
+http://www.oracle.com/us/products/export/export-regulations-345813.html
+for more details on each of these libraries cryptography features.
[03/31] incubator-nifi git commit: NIFI-150: Added documentation
Posted by mc...@apache.org.
NIFI-150: Added documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/42566cd6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/42566cd6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/42566cd6
Branch: refs/heads/NIFI-182
Commit: 42566cd6aaa8396d83c547be74a0e77e8981ec02
Parents: a23c7f2
Author: Mark Payne <ma...@hotmail.com>
Authored: Thu Dec 18 17:25:22 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Thu Dec 18 17:25:22 2014 -0500
----------------------------------------------------------------------
nifi-docs/src/main/asciidoc/nifi-user-guide.adoc | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/42566cd6/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
index 931819d..55e2d09 100644
--- a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
@@ -394,6 +394,10 @@ image::comments-tab.png["Comments Tab"]
=== Additional Help
+Each Processor has the ability to provide additional documentation about its usage. This documentation can be found by right-clicking
+on the Processor and then selecting the `Usage' item from the context menu. Alternatively, clicking the `Help' link in the top-right
+corner of the application will provide a Help page with all of the Processors that are available. Clicking on the Processor in the list
+will then show its usage.
=== Connecting Components
@@ -440,7 +444,18 @@ value of `0 sec` indicates that the data will never expire.
NiFi provides two different configuration elements for back pressure. These thresholds indicate how much data should be
allowed to exist in the queue before the component that is the source of the Connection is no longer scheduled to run.
-This allows the system to avoid being overrun with data.
+This allows the system to avoid being overrun with data. The first option provided is the `Back pressure object threshold.'
+This is the number of FlowFiles that can be in the queue before back pressure is applied. The second configuration option
+is the `Back pressure data size threshold.' This specifies the maximum amount of data that should be queued up before
+applying back pressure. This value is configured by entering a number followed by a data size (`B` for bytes, `KB` for
+kilobytes, `MB` for megabytes, `GB` for gigabytes, or `TB` for terabytes).
+
+The right-hand side of the tab provides the ability to prioritize the data in queue so that higher priority data is
+processed first. Prioritizers can be dragged from the top (`Available prioritizers') to the bottom (`Selected prioritizers').
+Multiple prioritizers can be selected. The prioritizer that is at the top of the `Selected prioritizers' list is the highest
+priority. If two FlowFiles have the same value according to this prioritizer, the second prioritizer will determine which
+FlowFile to process first, and so on. If a prioritizer is no longer desired, it can then be dragged from the `Selected
+prioritizers' list to the `Available prioritizers' list.
[04/31] incubator-nifi git commit: NIFI-150: Added Apache License to
the output HTML
Posted by mc...@apache.org.
NIFI-150: Added Apache License to the output HTML
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/c213895c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/c213895c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/c213895c
Branch: refs/heads/NIFI-182
Commit: c213895cc2eb0372335f392cfcb4e3c3af101208
Parents: 42566cd
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 10:01:09 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 10:01:09 2014 -0500
----------------------------------------------------------------------
nifi-docs/pom.xml | 44 ++++
.../main/asciidoc/images/invalid-processor.png | Bin 0 -> 8187 bytes
.../main/asciidoc/images/valid-processor.png | Bin 0 -> 8090 bytes
.../src/main/asciidoc/nifi-user-guide.adoc | 242 ++++++++++---------
4 files changed, 173 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c213895c/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
index 1b74533..a8a546a 100644
--- a/nifi-docs/pom.xml
+++ b/nifi-docs/pom.xml
@@ -28,8 +28,52 @@
</executions>
<configuration>
<backend>html</backend>
+ <attributes>
+ <skipFrontMatter>true</skipFrontMatter>
+ </attributes>
</configuration>
</plugin>
+
+ <!-- This plugin is used to insert the Apache License into the output HMTL because
+ AsciiDoc doesn't appear to provide a mechanism for doing this. -->
+ <plugin>
+ <groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <version>1.5.3</version>
+ <executions>
+ <execution>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>replace</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <file>target/generated-docs/nifi-user-guide.html</file>
+ <regexFlags>
+ <regexFlag>DOTALL</regexFlag>
+ </regexFlags>
+ <token>(.*)</token>
+ <value><!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+$1
+ </value>
+ </configuration>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c213895c/nifi-docs/src/main/asciidoc/images/invalid-processor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/invalid-processor.png b/nifi-docs/src/main/asciidoc/images/invalid-processor.png
new file mode 100644
index 0000000..37117b4
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/invalid-processor.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c213895c/nifi-docs/src/main/asciidoc/images/valid-processor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/valid-processor.png b/nifi-docs/src/main/asciidoc/images/valid-processor.png
new file mode 100644
index 0000000..d6102b7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/valid-processor.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c213895c/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
index 55e2d09..8f01c01 100644
--- a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
@@ -1,7 +1,23 @@
-NiFi User Guide
-===============
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+NiFi User Guide (Draft - Preview Version)
+=========================================
Apache_NiFi_Team
-:toc:
+:toc: left
:icons:
@@ -10,103 +26,103 @@ Terminology
-----------
*DataFlow Manager*: A DataFlow Manager (DFM) is a NiFi user who has permissions to add, remove, and modify components of a NiFi dataflow.
-*FlowFile*: The FlowFile represents a single piece of data in NiFi. A FlowFile is made up of two components:
- FlowFile Attributes and FlowFile Content.
- Content is the data that is represented by the FlowFile. Attributes are key-value pairs that provide information or
- context about the data.
+*FlowFile*: The FlowFile represents a single piece of data in NiFi. A FlowFile is made up of two components:
+ FlowFile Attributes and FlowFile Content.
+ Content is the data that is represented by the FlowFile. Attributes are key-value pairs that provide information or
+ context about the data.
All FlowFiles have the following Standard Attributes:
-
+
- *uuid*: A unique identifier for the FlowFile
- *filename*: A human-readable filename that may be used when storing the data to disk or in an external service
- *path*: A hierarchically structured value that can be used when storing data to disk or an external service so that the data is not stored in a single directory
-*Processor*: The Processor is the NiFi component that is used to listen for incoming data; pull data from external sources;
+*Processor*: The Processor is the NiFi component that is used to listen for incoming data; pull data from external sources;
publish data to external sources; and route, transform, or extract information from FlowFiles.
-*Relationship*: Each Processor has zero or more Relationships defined for it. These Relationships are named to indicate the result of processing a FlowFile.
- After a Processor has finished processing a FlowFile, it will route (or “transfer”) the FlowFile to one of the Relationships.
- A DataFlow Manager is then able to connect each of these Relationships to other components in order to specify where the FlowFile should
+*Relationship*: Each Processor has zero or more Relationships defined for it. These Relationships are named to indicate the result of processing a FlowFile.
+ After a Processor has finished processing a FlowFile, it will route (or “transfer”) the FlowFile to one of the Relationships.
+ A DataFlow Manager is then able to connect each of these Relationships to other components in order to specify where the FlowFile should
go next under each potential processing result.
-*Connection*: A DataFlow Manager creates an automated dataflow by dragging components from the Components part of the NiFi toolbar to the canvas
- and then connecting the components together via Connections. Each connection consists of one or more Relationships.
- For each Connection that is drawn, a DataFlow Manager can determine which Relationships should be used for the Connection.
- This allows data to be routed in different ways based on its processing outcome. Each connection houses a FlowFile Queue.
+*Connection*: A DataFlow Manager creates an automated dataflow by dragging components from the Components part of the NiFi toolbar to the canvas
+ and then connecting the components together via Connections. Each connection consists of one or more Relationships.
+ For each Connection that is drawn, a DataFlow Manager can determine which Relationships should be used for the Connection.
+ This allows data to be routed in different ways based on its processing outcome. Each connection houses a FlowFile Queue.
When a FlowFile is transferred to a particular Relationship, it is added to the queue belonging to the associated Connection.
-
+
*Funnel*: A funnel is a NiFi component that is used to combine the data from several Connections into a single Connection.
-*Process Group*: When a dataflow becomes complex, it often is beneficial to reason about the dataflow at a higher, more abstract level.
- NiFi allows multiple components, such as Processors, to be grouped together into a Process Group.
- The NiFi User Interface then makes it easy for a DataFlow Manager to connect together multiple Process Groups into a logical dataflow,
+*Process Group*: When a dataflow becomes complex, it often is beneficial to reason about the dataflow at a higher, more abstract level.
+ NiFi allows multiple components, such as Processors, to be grouped together into a Process Group.
+ The NiFi User Interface then makes it easy for a DataFlow Manager to connect together multiple Process Groups into a logical dataflow,
as well as allowing the DataFlow Manager to enter a Process Group in order to see and manipulate the components within the Process Group.
-*Port*: Dataflows that are constructed using one or more Process Groups need a way to connect a Process Group to other dataflow components.
+*Port*: Dataflows that are constructed using one or more Process Groups need a way to connect a Process Group to other dataflow components.
This is achieved by using Ports. A DataFlow Manager can add any number of Input Ports and Output Ports to a Process Group and name these ports appropriately.
-*Remote Process Group*: Just as data is transferred into and out of a Process Group, it is sometimes necessary to transfer data from one instance of NiFi to another.
- While NiFi provides many different mechanisms for transferring data from one system to another, Remote Process Groups are often the easiest way to accomplish
+*Remote Process Group*: Just as data is transferred into and out of a Process Group, it is sometimes necessary to transfer data from one instance of NiFi to another.
+ While NiFi provides many different mechanisms for transferring data from one system to another, Remote Process Groups are often the easiest way to accomplish
this if transferring data to another instance of NiFi.
-*Bulletin*: The NiFi User Interface provides a significant amount of monitoring and feedback about the current status of the application.
- In addition to rolling statistics and the current status that are provided for each component, components are able to report Bulletins.
- Whenever a component reports a Bulletin, an icon is displayed on that component (or on the Status bar near the top of the page, for System-Level Bulletins).
- Using the mouse to hover over that icon will provide a tool-tip that shows the time and severity (Debug, Info, Warning, Error) of the bulletin,
- as well as the message of the Bulletin.
+*Bulletin*: The NiFi User Interface provides a significant amount of monitoring and feedback about the current status of the application.
+ In addition to rolling statistics and the current status that are provided for each component, components are able to report Bulletins.
+ Whenever a component reports a Bulletin, an icon is displayed on that component (or on the Status bar near the top of the page, for System-Level Bulletins).
+ Using the mouse to hover over that icon will provide a tool-tip that shows the time and severity (Debug, Info, Warning, Error) of the bulletin,
+ as well as the message of the Bulletin.
Bulletins from all components can also be viewed and filtered in the Bulletins Page, available in the Management Toolbar.
-*Template*: Often times, a dataflow is comprised of many sub-flows that could be reused. NiFi allows DataFlow Managers to select a part of the dataflow
- (or the entire dataflow) and create a Template. This Template is given a name can then be dragged onto the canvas just like the other components.
- As a result, several components be combined together to make a larger building block from which to create a dataflow.
+*Template*: Often times, a dataflow is comprised of many sub-flows that could be reused. NiFi allows DataFlow Managers to select a part of the dataflow
+ (or the entire dataflow) and create a Template. This Template is given a name can then be dragged onto the canvas just like the other components.
+ As a result, several components be combined together to make a larger building block from which to create a dataflow.
These templates can also be exported as XML and imported into another NiFi instance, allowing these building blocks to be shared.
-
-
+
+
NiFi User Interface
-------------------
-The NiFi User Interface (UI) provides mechanisms for creating automated dataflows, as well as visualizing,
-editing, monitoring, and administering those dataflows. The UI can be broken down into several different segments,
-each responsible for different functionality of the application. We will begin by looking at screenshots of the
-application and labeling the different segments of the UI. We will provide a brief explanation of the purpose of each segment.
+The NiFi User Interface (UI) provides mechanisms for creating automated dataflows, as well as visualizing,
+editing, monitoring, and administering those dataflows. The UI can be broken down into several different segments,
+each responsible for different functionality of the application. We will begin by looking at screenshots of the
+application and labeling the different segments of the UI. We will provide a brief explanation of the purpose of each segment.
Then, in the following sections of this document, we will discuss each of those segments in greater detail.
-When the application is started, by default, the user is able to navigate to the User Interface by going to
-`http://<hostname>:8080/nifi` in a web browser. There are no permissions configured, by default, so anyone is
+When the application is started, by default, the user is able to navigate to the User Interface by going to
+`http://<hostname>:8080/nifi` in a web browser. There are no permissions configured, by default, so anyone is
able to view and modify the dataflow. For information on securing the system, see Systems Administrator guide.
When a DataFlow Manager navigates to the UI for the first time, a blank canvas is provided on which a dataflow can be built:
image::new-flow.png["Empty Flow"]
-Along the top of the of the screen is a toolbar that contains several of these segments.
-To the left is the Components Toolbar. This toolbar consists of the different components that can be dragged onto the canvas.
+Along the top of the of the screen is a toolbar that contains several of these segments.
+To the left is the Components Toolbar. This toolbar consists of the different components that can be dragged onto the canvas.
-Next to the Components Toolbar is the Actions Toolbar. This toolbar consists of buttons to manipulate the existing
-components on the graph. Following the Actions Toolbar is the Search Toolbar. This toolbar consists of a single
-Search field that allows users to easily find components on the graph. Users are able to search by component name,
+Next to the Components Toolbar is the Actions Toolbar. This toolbar consists of buttons to manipulate the existing
+components on the graph. Following the Actions Toolbar is the Search Toolbar. This toolbar consists of a single
+Search field that allows users to easily find components on the graph. Users are able to search by component name,
type, identifier, and configuration properties.
-Finally, the Management Toolbar sits to the right-hand side of the screen. This toolbar consists of buttons that are
-of use to DataFlow Managers to manage the flow as well as administrators who may use this section to manage user access
+Finally, the Management Toolbar sits to the right-hand side of the screen. This toolbar consists of buttons that are
+of use to DataFlow Managers to manage the flow as well as administrators who may use this section to manage user access
and configure system properties, such as how many system resources should be provided to the application.
image::nifi-toolbar-components.png["NiFi Components Toolbar"]
-Next, we have segments that provide capabilities to easily navigate around the graph. On the left-hand side is a toolbar that
-provides the ability to pan around the graph and zoom in and out. On the right-hand side is a “Birds-Eye View” of the dataflow.
-This provides a high-level view of the dataflow and allows the user to quickly and easily pan across large portions of the dataflow.
-Along the top of the screen is a trail of breadcrumbs. As users navigate into and out of Process Groups, the breadcrumbs show
-the depth in the flow and each Process Group that was entered to reach this depth. Each of the Process Groups listed in the breadcrumbs
+Next, we have segments that provide capabilities to easily navigate around the graph. On the left-hand side is a toolbar that
+provides the ability to pan around the graph and zoom in and out. On the right-hand side is a “Birds-Eye View” of the dataflow.
+This provides a high-level view of the dataflow and allows the user to quickly and easily pan across large portions of the dataflow.
+Along the top of the screen is a trail of breadcrumbs. As users navigate into and out of Process Groups, the breadcrumbs show
+the depth in the flow and each Process Group that was entered to reach this depth. Each of the Process Groups listed in the breadcrumbs
is a link that will take you back up to that level in the flow.
image::nifi-navigation.png["NiFi Navigation"]
-Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist in the graph in
-each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the graph in each state
-(Transmitting, Not Transmitting), the number of threads that are currently active in the flow, the amount of data that currently
-exists in the flow, and the timestamp at which all of this information was last refreshed. If there are any System-Level bulletins,
-these are shown in the Status bar as well. Additionally, if the instance of NiFi is clustered, the Status bar shows many nodes
+Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist in the graph in
+each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the graph in each state
+(Transmitting, Not Transmitting), the number of threads that are currently active in the flow, the amount of data that currently
+exists in the flow, and the timestamp at which all of this information was last refreshed. If there are any System-Level bulletins,
+these are shown in the Status bar as well. Additionally, if the instance of NiFi is clustered, the Status bar shows many nodes
are in the cluster and how many are currently connected.
image::status-bar.png["NiFi Status Bar"]
@@ -118,100 +134,100 @@ image::status-bar.png["NiFi Status Bar"]
Building a DataFlow
-------------------
-A DataFlow Manager (DFM) is able to build an automated dataflow using the NiFi User Interface (UI). This is accomplished
-by dragging components from the toolbar to the canvas, configuring the components to meet specific needs, and connecting
+A DataFlow Manager (DFM) is able to build an automated dataflow using the NiFi User Interface (UI). This is accomplished
+by dragging components from the toolbar to the canvas, configuring the components to meet specific needs, and connecting
the components together.
=== Adding Components to the Canvas
-In the User Interface section above, we outlined the different segments of the UI and pointed out a Components Toolbar.
+In the User Interface section above, we outlined the different segments of the UI and pointed out a Components Toolbar.
Here, we will look at each of the Components in that toolbar:
image::components.png["Components"]
image:iconProcessor.png["Processor", width=32]
-*Processor*: The Processor is the most commonly used component, as it is responsible for data ingress, egress, routing, and
- manipulating. There are many different types of Processors. In fact, this is a very common Extension Point in NiFi,
- meaning that many vendors may implement their own Processors to perform whatever functions are necessary for their use case.
+*Processor*: The Processor is the most commonly used component, as it is responsible for data ingress, egress, routing, and
+ manipulating. There are many different types of Processors. In fact, this is a very common Extension Point in NiFi,
+ meaning that many vendors may implement their own Processors to perform whatever functions are necessary for their use case.
When a Processor is dragged onto the graph, the user is presented with a dialog to choose which type of Processor to use:
image::add-processor.png["Add Processor Dialog"]
-In the top-right corner, the user is able to filter the list based on the Processor Type or the Tags associated with a Processor.
-Processor developers have the ability to add Tags to their Processors. These tags are used in this dialog for filtering and are
-displayed on the left-hand side in a Tag Cloud. The more Processors that exist with a particular Tag, the larger the Tag appears
-in the Tag Cloud. Clicking a Tag in the Cloud will filter the available Processors to only those that contain that Tag. If multiple
-Tags are selected, only those Processors that contain all of those Tags are shown. For example, if we want to show only those
+In the top-right corner, the user is able to filter the list based on the Processor Type or the Tags associated with a Processor.
+Processor developers have the ability to add Tags to their Processors. These tags are used in this dialog for filtering and are
+displayed on the left-hand side in a Tag Cloud. The more Processors that exist with a particular Tag, the larger the Tag appears
+in the Tag Cloud. Clicking a Tag in the Cloud will filter the available Processors to only those that contain that Tag. If multiple
+Tags are selected, only those Processors that contain all of those Tags are shown. For example, if we want to show only those
Processors that allow us to ingest data via HTTP, we can select both the `http` Tag and the `ingest` Tag:
image::add-processor-with-tag-cloud.png["Add Processor with Tag Cloud"]
-Clicking the `Add` button or double-clicking on a Processor Type will add the selected Processor to the canvas at the
+Clicking the `Add` button or double-clicking on a Processor Type will add the selected Processor to the canvas at the
location that it was dropped.
image:iconInputPort.png["Input Port", width=32]
-*Input Port*: Input Ports provide a mechanism for transferring data into a Process Group. When an Input Port is dragged
+*Input Port*: Input Ports provide a mechanism for transferring data into a Process Group. When an Input Port is dragged
onto the canvas, the DFM is prompted to name the Port. All Ports within a Process Group must have unique names.
-All components exist only within a Process Group. When a user navigates to the NiFi page, the user is placed in the
-Root Progress Group. If the Input Port is dragged onto the Root Progress Group, the Input Port provides a mechanism
-to receive data from remote instances of NiFi. In this case, the Input Port can be configured to restrict access to
+All components exist only within a Process Group. When a user navigates to the NiFi page, the user is placed in the
+Root Progress Group. If the Input Port is dragged onto the Root Progress Group, the Input Port provides a mechanism
+to receive data from remote instances of NiFi. In this case, the Input Port can be configured to restrict access to
appropriate users.
image:iconOutputPort.png["Output Port", width=32]
-*Output Port*: Output Ports provide a mechanism for transferring data from a Process Group back to destination outside
-of the Process Group. When an Output Port is dragged onto the canvas, the DFM is prompted to name the Port. All Ports
+*Output Port*: Output Ports provide a mechanism for transferring data from a Process Group back to destination outside
+of the Process Group. When an Output Port is dragged onto the canvas, the DFM is prompted to name the Port. All Ports
within a Process Group must have unique names.
-If the Output Port is dragged onto the Root Process Group, the Output Port provides a mechanism for sending data to
-remote instances of NiFi. In this case, the Port acts as a queue. As remote instances of NiFi pull data from the port,
+If the Output Port is dragged onto the Root Process Group, the Output Port provides a mechanism for sending data to
+remote instances of NiFi. In this case, the Port acts as a queue. As remote instances of NiFi pull data from the port,
that data is removed from the queues of the incoming Connections.
image:iconProcessGroup.png["Process Group", width=32]
-*Process Group*: Process Groups can be used logically group a set of components so that the dataflow is easier to understand
-and maintain. When a Process Group is dragged onto the canvas, the DFM is prompted to name the Process Group. All Process
+*Process Group*: Process Groups can be used logically group a set of components so that the dataflow is easier to understand
+and maintain. When a Process Group is dragged onto the canvas, the DFM is prompted to name the Process Group. All Process
Groups within the same parent group must have unique names.
image:iconRemoteProcessGroup.png["Remote Process Group", width=32]
-*Remote Process Group*: Remote Process Groups appear and behave similar to Process Groups. However, the Remote Process Group (RPG)
-references a remote instance of NiFi. When an RPG is dragged onto the canvas, rather than being prompted for a name, the DFM
-is prompted for the URL of the remote NiFi instance. If the remote NiFi is a clustered instance, the URL that should be used
-is the URL of the remote instance's NiFi Cluster Manager (NCM). When data is transferred to a clustered instance of NiFi
-via an RPG, the RPG it will first connect to the remote instance's NCM to determine which nodes are in the cluster and
-how busy each node is. This information is then used to load balance the data that is pushed to each node. The remote NCM is
+*Remote Process Group*: Remote Process Groups appear and behave similar to Process Groups. However, the Remote Process Group (RPG)
+references a remote instance of NiFi. When an RPG is dragged onto the canvas, rather than being prompted for a name, the DFM
+is prompted for the URL of the remote NiFi instance. If the remote NiFi is a clustered instance, the URL that should be used
+is the URL of the remote instance's NiFi Cluster Manager (NCM). When data is transferred to a clustered instance of NiFi
+via an RPG, the RPG it will first connect to the remote instance's NCM to determine which nodes are in the cluster and
+how busy each node is. This information is then used to load balance the data that is pushed to each node. The remote NCM is
then interrogated periodically to ensure that any nodes that are dropped from the cluster and no longer sent to, any new nodes
will be added to the list of nodes, and to recalculate the load balancing based on each node's load.
image:iconFunnel.png["Funnel", width=32]
-*Funnel*: Funnels are used to combine the data from many Connections into a single Connection. This has two advantages.
-First, if many Connections are created with the same destination, the canvas can become cluttered if those Connections
-have to span a large space. By funneling these Connections into a single Connection, that single Connection can then be
-drawn to span that large space instead. Secondly, Connections can be configured with FlowFile Prioritizers. Data from
-several Connections can be funneled into a single Connection, providing the ability to Prioritize all of the data on that
+*Funnel*: Funnels are used to combine the data from many Connections into a single Connection. This has two advantages.
+First, if many Connections are created with the same destination, the canvas can become cluttered if those Connections
+have to span a large space. By funneling these Connections into a single Connection, that single Connection can then be
+drawn to span that large space instead. Secondly, Connections can be configured with FlowFile Prioritizers. Data from
+several Connections can be funneled into a single Connection, providing the ability to Prioritize all of the data on that
one Connection, rather than prioritizing the data on each Connection independently.
image:iconTemplate.png["Template", width=32]
-*Template*: Templates can be created by DataFlow Managers from sections of the flow, or they can be imported from other
-dataflows. These Templates provide larger building blocks for creating a complex flow quickly. When the Template is
+*Template*: Templates can be created by DataFlow Managers from sections of the flow, or they can be imported from other
+dataflows. These Templates provide larger building blocks for creating a complex flow quickly. When the Template is
dragged onto the canvas, the DFM is provided a dialog to choose which Template to add to the canvas:
image::instantiate-template.png["Instantiate Template Dialog"]
-Clicking the drop-down box shows all available Templates. Any Template that was created with a description will show an
+Clicking the drop-down box shows all available Templates. Any Template that was created with a description will show an
icon indicating that there is more information. Hovering over the icon with the mouse will show this description:
image::instantiate-template-description.png["Instantiate Template Dialog"]
@@ -220,9 +236,9 @@ image::instantiate-template-description.png["Instantiate Template Dialog"]
image:iconLabel.png["Label"]
-*Label*: Labels are used to provide documentation to parts of a dataflow. When a Label is dropped onto the canvas,
-it is created with a default size. The Label can then be resized by dragging the handle in the bottom-right corner.
-The Label has no text when initially created. The text of the Label can be added by right-clicking on the Label and
+*Label*: Labels are used to provide documentation to parts of a dataflow. When a Label is dropped onto the canvas,
+it is created with a default size. The Label can then be resized by dragging the handle in the bottom-right corner.
+The Label has no text when initially created. The text of the Label can be added by right-clicking on the Label and
choosing `Configure...`
@@ -257,7 +273,7 @@ Below the Name configuration, the Processor's unique identifier is displayed alo
values cannot be modified.
Next are two dialogues for configuring `Penalty duration' and `Yield duration'. During the normal course of processing a
-piece of data (a FlowFile), an event may occur that indicates that the data cannot be processed at this time but the
+piece of data (a FlowFile), an event may occur that indicates that the data cannot be processed at this time but the
data may be processable at a later time. When this occurs, the Processor may choose to Penalize the FlowFile. This will
prevent the FlowFile from being Processed for some period of time. For example, if the Processor is to push the data
to a remote service, but the remote service already has a file with the same name as the filename that the Processor
@@ -314,8 +330,8 @@ The first configuration option is the Scheduling Strategy. There are three optio
. Day of Week
. Year
+
-The value for each of these fields should be a number, range, or increment.
-Range here refers to a syntax of <number>-<number>.
+The value for each of these fields should be a number, range, or increment.
+Range here refers to a syntax of <number>-<number>.
For example,the Seconds field could be set to 0-30, meaning that the Processor should only be scheduled if the time is 0 to 30 seconds
after the minute. Additionally, a value of `*` indicates that all values are valid for this field. Multiple values can also
be entered using a `,` as a separator: `0,5,10,15,30`.
@@ -327,7 +343,7 @@ For the Month field, valid values are 1 (January) through 12 (December).
For the Day of Week field, valid values are 1 (Sunday) through 7 (Saturday). Additionally, a value of `L` may be appended to one of these
values to indicate the last occurrence of this day in the month. For example, `1L` can be used to indicate the last Monday of the month.
-
+
Next, the Scheduling Tab provides a configuration option named `Concurrent tasks.' This controls how many threads the Processor
will use. Said a different way, this controls how many FlowFiles should be processed by this Processor at the same time. Increasing
this value will typically allow the Processor to handle more data in the same amount of time. However, it does this by using system
@@ -339,7 +355,7 @@ The `Run schedule` dictates how often this Processor should be scheduled to run.
Scheduling Strategy (see above). If using the Event driven Scheduling Strategy, this field is not available. When using the Timer driven
Scheduling Strategy, this value is a time duration specified by a number followed by a time unit. For example, `1 second` or `5 mins`.
The default value of `0 sec` means that the Processor should run as often as possible as long as it has data to process. This is true
-for any time duration of 0, regardless of the time unit (i.e., `0 sec`, `0 mins`, `0 days`). For an explanation of values that are
+for any time duration of 0, regardless of the time unit (i.e., `0 sec`, `0 mins`, `0 days`). For an explanation of values that are
applicable for the CRON driven Scheduling Strategy, see the description of the CRON driven Scheduling Strategy itself.
The right-hand side of the tab contains a slider for choosing the `Run duration.' This controls how long the Processor should be scheduled
@@ -359,12 +375,12 @@ must define which Properties make sense for its use case. Below, we see the Prop
image::properties-tab.png["Properties Tab"]
-This Processor, by default, has only a single property: `Routing Strategy.' The default value is `Route on Property name.' Next to
+This Processor, by default, has only a single property: `Routing Strategy.' The default value is `Route on Property name.' Next to
the name of this property is a small question-mark symbol (
image:iconInfo.png["Question Mark"]
-). This help symbol is seen in other places throughout the application, as well, and indicates that more information is available.
-Hovering over this symbol with the mouse will provide additional details about the property and the default value, as well as
-historical values that have been set for the Property.
+). This help symbol is seen in other places throughout the application, as well, and indicates that more information is available.
+Hovering over this symbol with the mouse will provide additional details about the property and the default value, as well as
+historical values that have been set for the Property.
Clicking on the value for the property will allow a DFM to change the value. Depending on the values that are allowed for the property,
the user is either provided a drop-down from which to choose a value or is given a text area to type a value:
@@ -415,7 +431,7 @@ This Connection bubble can then be dragged from this component to another compon
==== Details Tab
-The Details Tab provides information about the source and destination components, including the component name, the
+The Details Tab provides information about the source and destination components, including the component name, the
component type, and the Process Group in which the component lives:
image::create-connection.png["Create Connection"]
@@ -442,15 +458,17 @@ In this case, the expiration can be used in conjunction with Prioritizers to ens
processed first and then anything that cannot be processed within one hour, for example, can be dropped. The default
value of `0 sec` indicates that the data will never expire.
+
NiFi provides two different configuration elements for back pressure. These thresholds indicate how much data should be
-allowed to exist in the queue before the component that is the source of the Connection is no longer scheduled to run.
-This allows the system to avoid being overrun with data. The first option provided is the `Back pressure object threshold.'
+allowed to exist in the queue before the component that is the source of the Connection is no longer scheduled to run.
+This allows the system to avoid being overrun with data. The first option provided is the ``Back pressure object threshold.''
This is the number of FlowFiles that can be in the queue before back pressure is applied. The second configuration option
-is the `Back pressure data size threshold.' This specifies the maximum amount of data that should be queued up before
-applying back pressure. This value is configured by entering a number followed by a data size (`B` for bytes, `KB` for
+is the ``Back pressure data size threshold.''
+This specifies the maximum amount of data that should be queued up before
+applying back pressure. This value is configured by entering a number followed by a data size (`B` for bytes, `KB` for
kilobytes, `MB` for megabytes, `GB` for gigabytes, or `TB` for terabytes).
-The right-hand side of the tab provides the ability to prioritize the data in queue so that higher priority data is
+The right-hand side of the tab provides the ability to prioritize the data in queue so that higher priority data is
processed first. Prioritizers can be dragged from the top (`Available prioritizers') to the bottom (`Selected prioritizers').
Multiple prioritizers can be selected. The prioritizer that is at the top of the `Selected prioritizers' list is the highest
priority. If two FlowFiles have the same value according to this prioritizer, the second prioritizer will determine which
@@ -459,10 +477,9 @@ prioritizers' list to the `Available prioritizers' list.
-
=== Processor Validation
-
+Before trying to start a Processor, it's important to make sure that the Processor's configuration is valid.
@@ -531,4 +548,3 @@ prioritizers' list to the `Available prioritizers' list.
=== Viewing FlowFile Lineage
==== Find Parents
==== Expanding an Event
-
[17/31] incubator-nifi git commit: NIFI-186 fixed extra + and made
default mem sizes equal
Posted by mc...@apache.org.
NIFI-186 fixed extra + and made default mem sizes equal
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/c0e4618d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/c0e4618d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/c0e4618d
Branch: refs/heads/NIFI-182
Commit: c0e4618d4233a2deb70e0ed44e2a936149f740cc
Parents: 21182ac
Author: joewitt <jo...@apache.org>
Authored: Fri Dec 19 17:29:43 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Fri Dec 19 17:29:43 2014 -0500
----------------------------------------------------------------------
.../framework/resources/src/main/resources/conf/bootstrap.conf | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c0e4618d/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf b/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
index 6208560..a34fa28 100644
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
+++ b/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
@@ -15,7 +15,7 @@ graceful.shutdown.seconds=20
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
-java.arg.2=-Xms256m
+java.arg.2=-Xms512m
java.arg.3=-Xmx512m
# Enable Remote Debugging
@@ -29,6 +29,6 @@ java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with
# many classes loaded in the JVM.
-java.arg.7=-XX:+ReservedCodeCacheSize=256m
+java.arg.7=-XX:ReservedCodeCacheSize=256m
java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m
java.arg.9=-XX:+UseCodeCacheFlushing
[24/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingInputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingInputStream.java
deleted file mode 100644
index 792cc32..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingInputStream.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-public class ByteCountingInputStream extends InputStream {
-
- private final InputStream in;
- private long bytesRead = 0L;
- private long bytesSkipped = 0L;
-
- private long bytesSinceMark = 0L;
-
- public ByteCountingInputStream(final InputStream in) {
- this.in = in;
- }
-
- @Override
- public int read() throws IOException {
- final int fromSuper = in.read();
- if (fromSuper >= 0) {
- bytesRead++;
- bytesSinceMark++;
- }
- return fromSuper;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- final int fromSuper = in.read(b, off, len);
- if (fromSuper >= 0) {
- bytesRead += fromSuper;
- bytesSinceMark += fromSuper;
- }
-
- return fromSuper;
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- return read(b, 0, b.length);
- }
-
- @Override
- public long skip(final long n) throws IOException {
- final long skipped = in.skip(n);
- if (skipped >= 0) {
- bytesSkipped += skipped;
- bytesSinceMark += skipped;
- }
- return skipped;
- }
-
- public long getBytesRead() {
- return bytesRead;
- }
-
- public long getBytesSkipped() {
- return bytesSkipped;
- }
-
- public long getBytesConsumed() {
- return getBytesRead() + getBytesSkipped();
- }
-
- @Override
- public void mark(final int readlimit) {
- in.mark(readlimit);
-
- bytesSinceMark = 0L;
- }
-
- @Override
- public boolean markSupported() {
- return in.markSupported();
- }
-
- @Override
- public void reset() throws IOException {
- in.reset();
- bytesRead -= bytesSinceMark;
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingOutputStream.java
deleted file mode 100644
index c7b77ff..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ByteCountingOutputStream.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-public class ByteCountingOutputStream extends OutputStream {
-
- private final OutputStream out;
- private long bytesWritten = 0L;
-
- public ByteCountingOutputStream(final OutputStream out) {
- this.out = out;
- }
-
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- bytesWritten++;
- }
-
- @Override
- public void write(byte[] b) throws IOException {
- write(b, 0, b.length);
- }
-
- ;
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- out.write(b, off, len);
- bytesWritten += len;
- }
-
- public long getBytesWritten() {
- return bytesWritten;
- }
-
- @Override
- public void flush() throws IOException {
- out.flush();
- }
-
- @Override
- public void close() throws IOException {
- out.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/DataOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/DataOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/DataOutputStream.java
deleted file mode 100644
index 6af06d3..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/DataOutputStream.java
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.DataOutput;
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.UTFDataFormatException;
-
-/**
- * This class is different from java.io.DataOutputStream in that it does
- * synchronize on its methods.
- */
-public class DataOutputStream extends FilterOutputStream implements DataOutput {
-
- /**
- * The number of bytes written to the data output stream so far. If this
- * counter overflows, it will be wrapped to Integer.MAX_VALUE.
- */
- protected int written;
-
- /**
- * bytearr is initialized on demand by writeUTF
- */
- private byte[] bytearr = null;
-
- /**
- * Creates a new data output stream to write data to the specified
- * underlying output stream. The counter <code>written</code> is set to
- * zero.
- *
- * @param out the underlying output stream, to be saved for later use.
- * @see java.io.FilterOutputStream#out
- */
- public DataOutputStream(OutputStream out) {
- super(out);
- }
-
- /**
- * Increases the written counter by the specified value until it reaches
- * Integer.MAX_VALUE.
- */
- private void incCount(int value) {
- int temp = written + value;
- if (temp < 0) {
- temp = Integer.MAX_VALUE;
- }
- written = temp;
- }
-
- /**
- * Writes the specified byte (the low eight bits of the argument
- * <code>b</code>) to the underlying output stream. If no exception is
- * thrown, the counter <code>written</code> is incremented by
- * <code>1</code>.
- * <p>
- * Implements the <code>write</code> method of <code>OutputStream</code>.
- *
- * @param b the <code>byte</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- incCount(1);
- }
-
- /**
- * Writes <code>len</code> bytes from the specified byte array starting at
- * offset <code>off</code> to the underlying output stream. If no exception
- * is thrown, the counter <code>written</code> is incremented by
- * <code>len</code>.
- *
- * @param b the data.
- * @param off the start offset in the data.
- * @param len the number of bytes to write.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public void write(byte b[], int off, int len) throws IOException {
- out.write(b, off, len);
- incCount(len);
- }
-
- /**
- * Flushes this data output stream. This forces any buffered output bytes to
- * be written out to the stream.
- * <p>
- * The <code>flush</code> method of <code>DataOutputStream</code> calls the
- * <code>flush</code> method of its underlying output stream.
- *
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- * @see java.io.OutputStream#flush()
- */
- @Override
- public void flush() throws IOException {
- out.flush();
- }
-
- /**
- * Writes a <code>boolean</code> to the underlying output stream as a 1-byte
- * value. The value <code>true</code> is written out as the value
- * <code>(byte)1</code>; the value <code>false</code> is written out as the
- * value <code>(byte)0</code>. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>1</code>.
- *
- * @param v a <code>boolean</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeBoolean(boolean v) throws IOException {
- out.write(v ? 1 : 0);
- incCount(1);
- }
-
- /**
- * Writes out a <code>byte</code> to the underlying output stream as a
- * 1-byte value. If no exception is thrown, the counter <code>written</code>
- * is incremented by <code>1</code>.
- *
- * @param v a <code>byte</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeByte(int v) throws IOException {
- out.write(v);
- incCount(1);
- }
-
- /**
- * Writes a <code>short</code> to the underlying output stream as two bytes,
- * high byte first. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>2</code>.
- *
- * @param v a <code>short</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeShort(int v) throws IOException {
- out.write((v >>> 8) & 0xFF);
- out.write((v) & 0xFF);
- incCount(2);
- }
-
- /**
- * Writes a <code>char</code> to the underlying output stream as a 2-byte
- * value, high byte first. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>2</code>.
- *
- * @param v a <code>char</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeChar(int v) throws IOException {
- out.write((v >>> 8) & 0xFF);
- out.write((v) & 0xFF);
- incCount(2);
- }
-
- /**
- * Writes an <code>int</code> to the underlying output stream as four bytes,
- * high byte first. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>4</code>.
- *
- * @param v an <code>int</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeInt(int v) throws IOException {
- out.write((v >>> 24) & 0xFF);
- out.write((v >>> 16) & 0xFF);
- out.write((v >>> 8) & 0xFF);
- out.write((v) & 0xFF);
- incCount(4);
- }
-
- private final byte writeBuffer[] = new byte[8];
-
- /**
- * Writes a <code>long</code> to the underlying output stream as eight
- * bytes, high byte first. In no exception is thrown, the counter
- * <code>written</code> is incremented by <code>8</code>.
- *
- * @param v a <code>long</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeLong(long v) throws IOException {
- writeBuffer[0] = (byte) (v >>> 56);
- writeBuffer[1] = (byte) (v >>> 48);
- writeBuffer[2] = (byte) (v >>> 40);
- writeBuffer[3] = (byte) (v >>> 32);
- writeBuffer[4] = (byte) (v >>> 24);
- writeBuffer[5] = (byte) (v >>> 16);
- writeBuffer[6] = (byte) (v >>> 8);
- writeBuffer[7] = (byte) (v);
- out.write(writeBuffer, 0, 8);
- incCount(8);
- }
-
- /**
- * Converts the float argument to an <code>int</code> using the
- * <code>floatToIntBits</code> method in class <code>Float</code>, and then
- * writes that <code>int</code> value to the underlying output stream as a
- * 4-byte quantity, high byte first. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>4</code>.
- *
- * @param v a <code>float</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- * @see java.lang.Float#floatToIntBits(float)
- */
- @Override
- public final void writeFloat(float v) throws IOException {
- writeInt(Float.floatToIntBits(v));
- }
-
- /**
- * Converts the double argument to a <code>long</code> using the
- * <code>doubleToLongBits</code> method in class <code>Double</code>, and
- * then writes that <code>long</code> value to the underlying output stream
- * as an 8-byte quantity, high byte first. If no exception is thrown, the
- * counter <code>written</code> is incremented by <code>8</code>.
- *
- * @param v a <code>double</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- * @see java.lang.Double#doubleToLongBits(double)
- */
- @Override
- public final void writeDouble(double v) throws IOException {
- writeLong(Double.doubleToLongBits(v));
- }
-
- /**
- * Writes out the string to the underlying output stream as a sequence of
- * bytes. Each character in the string is written out, in sequence, by
- * discarding its high eight bits. If no exception is thrown, the counter
- * <code>written</code> is incremented by the length of <code>s</code>.
- *
- * @param s a string of bytes to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeBytes(String s) throws IOException {
- int len = s.length();
- for (int i = 0; i < len; i++) {
- out.write((byte) s.charAt(i));
- }
- incCount(len);
- }
-
- /**
- * Writes a string to the underlying output stream as a sequence of
- * characters. Each character is written to the data output stream as if by
- * the <code>writeChar</code> method. If no exception is thrown, the counter
- * <code>written</code> is incremented by twice the length of
- * <code>s</code>.
- *
- * @param s a <code>String</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.DataOutputStream#writeChar(int)
- * @see java.io.FilterOutputStream#out
- */
- @Override
- public final void writeChars(String s) throws IOException {
- int len = s.length();
- for (int i = 0; i < len; i++) {
- int v = s.charAt(i);
- out.write((v >>> 8) & 0xFF);
- out.write((v) & 0xFF);
- }
- incCount(len * 2);
- }
-
- /**
- * Writes a string to the underlying output stream using
- * <a href="DataInput.html#modified-utf-8">modified UTF-8</a>
- * encoding in a machine-independent manner.
- * <p>
- * First, two bytes are written to the output stream as if by the
- * <code>writeShort</code> method giving the number of bytes to follow. This
- * value is the number of bytes actually written out, not the length of the
- * string. Following the length, each character of the string is output, in
- * sequence, using the modified UTF-8 encoding for the character. If no
- * exception is thrown, the counter <code>written</code> is incremented by
- * the total number of bytes written to the output stream. This will be at
- * least two plus the length of <code>str</code>, and at most two plus
- * thrice the length of <code>str</code>.
- *
- * @param str a string to be written.
- * @exception IOException if an I/O error occurs.
- */
- @Override
- public final void writeUTF(String str) throws IOException {
- writeUTF(str, this);
- }
-
- /**
- * Writes a string to the specified DataOutput using
- * <a href="DataInput.html#modified-utf-8">modified UTF-8</a>
- * encoding in a machine-independent manner.
- * <p>
- * First, two bytes are written to out as if by the <code>writeShort</code>
- * method giving the number of bytes to follow. This value is the number of
- * bytes actually written out, not the length of the string. Following the
- * length, each character of the string is output, in sequence, using the
- * modified UTF-8 encoding for the character. If no exception is thrown, the
- * counter <code>written</code> is incremented by the total number of bytes
- * written to the output stream. This will be at least two plus the length
- * of <code>str</code>, and at most two plus thrice the length of
- * <code>str</code>.
- *
- * @param str a string to be written.
- * @param out destination to write to
- * @return The number of bytes written out.
- * @exception IOException if an I/O error occurs.
- */
- static int writeUTF(String str, DataOutput out) throws IOException {
- int strlen = str.length();
- int utflen = 0;
- int c, count = 0;
-
- /* use charAt instead of copying String to char array */
- for (int i = 0; i < strlen; i++) {
- c = str.charAt(i);
- if ((c >= 0x0001) && (c <= 0x007F)) {
- utflen++;
- } else if (c > 0x07FF) {
- utflen += 3;
- } else {
- utflen += 2;
- }
- }
-
- if (utflen > 65535) {
- throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes");
- }
-
- byte[] bytearr = null;
- if (out instanceof DataOutputStream) {
- DataOutputStream dos = (DataOutputStream) out;
- if (dos.bytearr == null || (dos.bytearr.length < (utflen + 2))) {
- dos.bytearr = new byte[(utflen * 2) + 2];
- }
- bytearr = dos.bytearr;
- } else {
- bytearr = new byte[utflen + 2];
- }
-
- bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF);
- bytearr[count++] = (byte) ((utflen) & 0xFF);
-
- int i = 0;
- for (i = 0; i < strlen; i++) {
- c = str.charAt(i);
- if (!((c >= 0x0001) && (c <= 0x007F))) {
- break;
- }
- bytearr[count++] = (byte) c;
- }
-
- for (; i < strlen; i++) {
- c = str.charAt(i);
- if ((c >= 0x0001) && (c <= 0x007F)) {
- bytearr[count++] = (byte) c;
-
- } else if (c > 0x07FF) {
- bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
- bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
- bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
- } else {
- bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
- bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
- }
- }
- out.write(bytearr, 0, utflen + 2);
- return utflen + 2;
- }
-
- /**
- * Returns the current value of the counter <code>written</code>, the number
- * of bytes written to this data output stream so far. If the counter
- * overflows, it will be wrapped to Integer.MAX_VALUE.
- *
- * @return the value of the <code>written</code> field.
- * @see java.io.DataOutputStream#written
- */
- public final int size() {
- return written;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/GZIPOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/GZIPOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/GZIPOutputStream.java
deleted file mode 100644
index 875b838..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/GZIPOutputStream.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * <p>
- * This class extends the {@link java.util.zip.GZIPOutputStream} by allowing the
- * constructor to provide a compression level, and uses a default value of 1,
- * rather than 5.
- * </p>
- */
-public class GZIPOutputStream extends java.util.zip.GZIPOutputStream {
-
- public static final int DEFAULT_COMPRESSION_LEVEL = 1;
-
- public GZIPOutputStream(final OutputStream out) throws IOException {
- this(out, DEFAULT_COMPRESSION_LEVEL);
- }
-
- public GZIPOutputStream(final OutputStream out, final int compressionLevel) throws IOException {
- super(out);
- def.setLevel(compressionLevel);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/LeakyBucketStreamThrottler.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/LeakyBucketStreamThrottler.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/LeakyBucketStreamThrottler.java
deleted file mode 100644
index 0ebe16d..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/LeakyBucketStreamThrottler.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class LeakyBucketStreamThrottler implements StreamThrottler {
-
- private final int maxBytesPerSecond;
- private final BlockingQueue<Request> requestQueue = new LinkedBlockingQueue<Request>();
- private final ScheduledExecutorService executorService;
- private final AtomicBoolean shutdown = new AtomicBoolean(false);
-
- public LeakyBucketStreamThrottler(final int maxBytesPerSecond) {
- this.maxBytesPerSecond = maxBytesPerSecond;
-
- executorService = Executors.newSingleThreadScheduledExecutor();
- final Runnable task = new Drain();
- executorService.scheduleAtFixedRate(task, 0, 1000, TimeUnit.MILLISECONDS);
- }
-
- @Override
- public void close() {
- this.shutdown.set(true);
-
- executorService.shutdown();
- try {
- // Should not take more than 2 seconds because we run every second. If it takes more than
- // 2 seconds, it is because the Runnable thread is blocking on a write; in this case,
- // we will just ignore it and return
- executorService.awaitTermination(2, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- }
- }
-
- @Override
- public OutputStream newThrottledOutputStream(final OutputStream toWrap) {
- return new OutputStream() {
- @Override
- public void write(final int b) throws IOException {
- write(new byte[]{(byte) b}, 0, 1);
- }
-
- @Override
- public void write(byte[] b) throws IOException {
- write(b, 0, b.length);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- final InputStream in = new ByteArrayInputStream(b, off, len);
- LeakyBucketStreamThrottler.this.copy(in, toWrap);
- }
-
- @Override
- public void close() throws IOException {
- toWrap.close();
- }
-
- @Override
- public void flush() throws IOException {
- toWrap.flush();
- }
- };
- }
-
- @Override
- public InputStream newThrottledInputStream(final InputStream toWrap) {
- return new InputStream() {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- @Override
- public int read() throws IOException {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream(1);
- LeakyBucketStreamThrottler.this.copy(toWrap, baos, 1L);
- if (baos.getBufferLength() < 1) {
- return -1;
- }
-
- return baos.getUnderlyingBuffer()[0] & 0xFF;
- }
-
- @Override
- public int read(final byte[] b) throws IOException {
- if(b.length == 0){
- return 0;
- }
- return read(b, 0, b.length);
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- baos.reset();
- final int copied = (int) LeakyBucketStreamThrottler.this.copy(toWrap, baos, len);
- if (copied == 0) {
- return -1;
- }
- System.arraycopy(baos.getUnderlyingBuffer(), 0, b, off, copied);
- return copied;
- }
-
- @Override
- public void close() throws IOException {
- toWrap.close();
- }
-
- @Override
- public int available() throws IOException {
- return toWrap.available();
- }
- };
- }
-
- @Override
- public long copy(final InputStream in, final OutputStream out) throws IOException {
- return copy(in, out, -1);
- }
-
- @Override
- public long copy(final InputStream in, final OutputStream out, final long maxBytes) throws IOException {
- long totalBytesCopied = 0;
- boolean finished = false;
- while (!finished) {
- final long requestMax = (maxBytes < 0) ? Long.MAX_VALUE : maxBytes - totalBytesCopied;
- final Request request = new Request(in, out, requestMax);
- boolean transferred = false;
- while (!transferred) {
- if (shutdown.get()) {
- throw new IOException("Throttler shutdown");
- }
-
- try {
- transferred = requestQueue.offer(request, 1000, TimeUnit.MILLISECONDS);
- } catch (final InterruptedException e) {
- throw new IOException("Interrupted", e);
- }
- }
-
- final BlockingQueue<Response> responseQueue = request.getResponseQueue();
- Response response = null;
- while (response == null) {
- try {
- if (shutdown.get()) {
- throw new IOException("Throttler shutdown");
- }
- response = responseQueue.poll(1000L, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- throw new IOException("Interrupted", e);
- }
- }
-
- if (!response.isSuccess()) {
- throw response.getError();
- }
-
- totalBytesCopied += response.getBytesCopied();
- finished = (response.getBytesCopied() == 0) || (totalBytesCopied >= maxBytes && maxBytes > 0);
- }
-
- return totalBytesCopied;
- }
-
- /**
- * This class is responsible for draining water from the leaky bucket. I.e.,
- * it actually moves the data
- */
- private class Drain implements Runnable {
-
- private final byte[] buffer;
-
- public Drain() {
- final int bufferSize = Math.min(4096, maxBytesPerSecond);
- buffer = new byte[bufferSize];
- }
-
- @Override
- public void run() {
- final long start = System.currentTimeMillis();
-
- int bytesTransferred = 0;
- while (bytesTransferred < maxBytesPerSecond) {
- final long maxMillisToWait = 1000 - (System.currentTimeMillis() - start);
- if (maxMillisToWait < 1) {
- return;
- }
-
- try {
- final Request request = requestQueue.poll(maxMillisToWait, TimeUnit.MILLISECONDS);
- if (request == null) {
- return;
- }
-
- final BlockingQueue<Response> responseQueue = request.getResponseQueue();
-
- final OutputStream out = request.getOutputStream();
- final InputStream in = request.getInputStream();
-
- try {
- final long requestMax = request.getMaxBytesToCopy();
- long maxBytesToTransfer;
- if (requestMax < 0) {
- maxBytesToTransfer = Math.min(buffer.length, maxBytesPerSecond - bytesTransferred);
- } else {
- maxBytesToTransfer = Math.min(requestMax,
- Math.min(buffer.length, maxBytesPerSecond - bytesTransferred));
- }
- maxBytesToTransfer = Math.max(1L, maxBytesToTransfer);
-
- final int bytesCopied = fillBuffer(in, maxBytesToTransfer);
- out.write(buffer, 0, bytesCopied);
-
- final Response response = new Response(true, bytesCopied);
- responseQueue.put(response);
- bytesTransferred += bytesCopied;
- } catch (final IOException e) {
- final Response response = new Response(e);
- responseQueue.put(response);
- }
- } catch (InterruptedException e) {
- }
- }
- }
-
- private int fillBuffer(final InputStream in, final long maxBytes) throws IOException {
- int bytesRead = 0;
- int len;
- while (bytesRead < maxBytes && (len = in.read(buffer, bytesRead, (int) Math.min(maxBytes - bytesRead, buffer.length - bytesRead))) > 0) {
- bytesRead += len;
- }
-
- return bytesRead;
- }
- }
-
- private static class Response {
-
- private final boolean success;
- private final IOException error;
- private final int bytesCopied;
-
- public Response(final boolean success, final int bytesCopied) {
- this.success = success;
- this.bytesCopied = bytesCopied;
- this.error = null;
- }
-
- public Response(final IOException error) {
- this.success = false;
- this.error = error;
- this.bytesCopied = -1;
- }
-
- public boolean isSuccess() {
- return success;
- }
-
- public IOException getError() {
- return error;
- }
-
- public int getBytesCopied() {
- return bytesCopied;
- }
- }
-
- private static class Request {
-
- private final OutputStream out;
- private final InputStream in;
- private final long maxBytesToCopy;
- private final BlockingQueue<Response> responseQueue;
-
- public Request(final InputStream in, final OutputStream out, final long maxBytesToCopy) {
- this.out = out;
- this.in = in;
- this.maxBytesToCopy = maxBytesToCopy;
- this.responseQueue = new LinkedBlockingQueue<Response>(1);
- }
-
- public BlockingQueue<Response> getResponseQueue() {
- return this.responseQueue;
- }
-
- public OutputStream getOutputStream() {
- return out;
- }
-
- public InputStream getInputStream() {
- return in;
- }
-
- public long getMaxBytesToCopy() {
- return maxBytesToCopy;
- }
-
- @Override
- public String toString() {
- return "Request[maxBytes=" + maxBytesToCopy + "]";
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableInputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableInputStream.java
deleted file mode 100644
index 1fbb093..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableInputStream.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.FilterInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Wraps and InputStream so that the underlying InputStream cannot be closed.
- * This is used so that the InputStream can be wrapped with yet another
- * InputStream and prevent the outer layer from closing the inner InputStream
- */
-public class NonCloseableInputStream extends FilterInputStream {
-
- private final InputStream toWrap;
-
- public NonCloseableInputStream(final InputStream toWrap) {
- super(toWrap);
- this.toWrap = toWrap;
- }
-
- @Override
- public int read() throws IOException {
- return toWrap.read();
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- return toWrap.read(b);
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- return toWrap.read(b, off, len);
- }
-
- @Override
- public void close() throws IOException {
- // do nothing
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableOutputStream.java
deleted file mode 100644
index 731e409..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NonCloseableOutputStream.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-public class NonCloseableOutputStream extends FilterOutputStream {
-
- private final OutputStream out;
-
- public NonCloseableOutputStream(final OutputStream out) {
- super(out);
- this.out = out;
- }
-
- @Override
- public void write(byte[] b) throws IOException {
- out.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- out.write(b, off, len);
- }
-
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- }
-
- @Override
- public void close() throws IOException {
- out.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NullOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NullOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NullOutputStream.java
deleted file mode 100644
index 60475d4..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/NullOutputStream.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * OutputStream that throws away all data, just like as if writing to /dev/null
- */
-public class NullOutputStream extends OutputStream {
-
- @Override
- public void write(final int b) throws IOException {
- }
-
- @Override
- public void write(final byte[] b) throws IOException {
- }
-
- @Override
- public void write(final byte[] b, int off, int len) throws IOException {
- }
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public void flush() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamThrottler.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamThrottler.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamThrottler.java
deleted file mode 100644
index 8c2aa80..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamThrottler.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-public interface StreamThrottler extends Closeable {
-
- long copy(InputStream in, OutputStream out) throws IOException;
-
- long copy(InputStream in, OutputStream out, long maxBytes) throws IOException;
-
- InputStream newThrottledInputStream(final InputStream toWrap);
-
- OutputStream newThrottledOutputStream(final OutputStream toWrap);
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamUtils.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamUtils.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamUtils.java
deleted file mode 100644
index 1596014..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/StreamUtils.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.nifi.io.exception.BytePatternNotFoundException;
-import org.apache.nifi.io.util.NonThreadSafeCircularBuffer;
-
-public class StreamUtils {
-
- public static long copy(final InputStream source, final OutputStream destination) throws IOException {
- final byte[] buffer = new byte[8192];
- int len;
- long totalCount = 0L;
- while ((len = source.read(buffer)) > 0) {
- destination.write(buffer, 0, len);
- totalCount += len;
- }
- return totalCount;
- }
-
- /**
- * Copies <code>numBytes</code> from <code>source</code> to
- * <code>destination</code>. If <code>numBytes</code> are not available from
- * <code>source</code>, throws EOFException
- *
- * @param source
- * @param destination
- * @param numBytes
- * @throws IOException
- */
- public static void copy(final InputStream source, final OutputStream destination, final long numBytes) throws IOException {
- final byte[] buffer = new byte[8192];
- int len;
- long bytesLeft = numBytes;
- while ((len = source.read(buffer, 0, (int) Math.min(bytesLeft, buffer.length))) > 0) {
- destination.write(buffer, 0, len);
- bytesLeft -= len;
- }
-
- if (bytesLeft > 0) {
- throw new EOFException("Attempted to copy " + numBytes + " bytes but only " + (numBytes - bytesLeft) + " bytes were available");
- }
- }
-
- /**
- * Reads data from the given input stream, copying it to the destination
- * byte array. If the InputStream has less data than the given byte array,
- * throws an EOFException
- *
- * @param source
- * @param destination
- * @throws IOException
- */
- public static void fillBuffer(final InputStream source, final byte[] destination) throws IOException {
- fillBuffer(source, destination, true);
- }
-
- /**
- * Reads data from the given input stream, copying it to the destination
- * byte array. If the InputStream has less data than the given byte array,
- * throws an EOFException if <code>ensureCapacity</code> is true and
- * otherwise returns the number of bytes copied
- *
- * @param source
- * @param destination
- * @param ensureCapacity whether or not to enforce that the InputStream have
- * at least as much data as the capacity of the destination byte array
- * @return
- * @throws IOException
- */
- public static int fillBuffer(final InputStream source, final byte[] destination, final boolean ensureCapacity) throws IOException {
- int bytesRead = 0;
- int len;
- while (bytesRead < destination.length) {
- len = source.read(destination, bytesRead, destination.length - bytesRead);
- if (len < 0) {
- if (ensureCapacity) {
- throw new EOFException();
- } else {
- break;
- }
- }
-
- bytesRead += len;
- }
-
- return bytesRead;
- }
-
- /**
- * Copies data from in to out until either we are out of data (returns null)
- * or we hit one of the byte patterns identified by the
- * <code>stoppers</code> parameter (returns the byte pattern matched). The
- * bytes in the stopper will be copied.
- *
- * @param in
- * @param out
- * @param maxBytes
- * @param stoppers
- * @return the byte array matched, or null if end of stream was reached
- * @throws IOException
- */
- public static byte[] copyInclusive(final InputStream in, final OutputStream out, final int maxBytes, final byte[]... stoppers) throws IOException {
- if (stoppers.length == 0) {
- return null;
- }
-
- final List<NonThreadSafeCircularBuffer> circularBuffers = new ArrayList<NonThreadSafeCircularBuffer>();
- for (final byte[] stopper : stoppers) {
- circularBuffers.add(new NonThreadSafeCircularBuffer(stopper));
- }
-
- long bytesRead = 0;
- while (true) {
- final int next = in.read();
- if (next == -1) {
- return null;
- } else if (maxBytes > 0 && ++bytesRead >= maxBytes) {
- throw new BytePatternNotFoundException("Did not encounter any byte pattern that was expected; data does not appear to be in the expected format");
- }
-
- out.write(next);
-
- for (final NonThreadSafeCircularBuffer circ : circularBuffers) {
- if (circ.addAndCompare((byte) next)) {
- return circ.getByteArray();
- }
- }
- }
- }
-
- /**
- * Copies data from in to out until either we are out of data (returns null)
- * or we hit one of the byte patterns identified by the
- * <code>stoppers</code> parameter (returns the byte pattern matched). The
- * byte pattern matched will NOT be copied to the output and will be un-read
- * from the input.
- *
- * @param in
- * @param out
- * @param maxBytes
- * @param stoppers
- * @return the byte array matched, or null if end of stream was reached
- * @throws IOException
- */
- public static byte[] copyExclusive(final InputStream in, final OutputStream out, final int maxBytes, final byte[]... stoppers) throws IOException {
- if (stoppers.length == 0) {
- return null;
- }
-
- int longest = 0;
- NonThreadSafeCircularBuffer longestBuffer = null;
- final List<NonThreadSafeCircularBuffer> circularBuffers = new ArrayList<NonThreadSafeCircularBuffer>();
- for (final byte[] stopper : stoppers) {
- final NonThreadSafeCircularBuffer circularBuffer = new NonThreadSafeCircularBuffer(stopper);
- if (stopper.length > longest) {
- longest = stopper.length;
- longestBuffer = circularBuffer;
- circularBuffers.add(0, circularBuffer);
- } else {
- circularBuffers.add(circularBuffer);
- }
- }
-
- long bytesRead = 0;
- while (true) {
- final int next = in.read();
- if (next == -1) {
- return null;
- } else if (maxBytes > 0 && bytesRead++ > maxBytes) {
- throw new BytePatternNotFoundException("Did not encounter any byte pattern that was expected; data does not appear to be in the expected format");
- }
-
- for (final NonThreadSafeCircularBuffer circ : circularBuffers) {
- if (circ.addAndCompare((byte) next)) {
- // The longest buffer has some data that may not have been written out yet; we need to make sure
- // that we copy out those bytes.
- final int bytesToCopy = longest - circ.getByteArray().length;
- for (int i = 0; i < bytesToCopy; i++) {
- final int oldestByte = longestBuffer.getOldestByte();
- if (oldestByte != -1) {
- out.write(oldestByte);
- longestBuffer.addAndCompare((byte) 0);
- }
- }
-
- return circ.getByteArray();
- }
- }
-
- if (longestBuffer.isFilled()) {
- out.write(longestBuffer.getOldestByte());
- }
- }
- }
-
- /**
- * Skips the specified number of bytes from the InputStream
- *
- * If unable to skip that number of bytes, throws EOFException
- *
- * @param stream
- * @param bytesToSkip
- * @throws IOException
- */
- public static void skip(final InputStream stream, final long bytesToSkip) throws IOException {
- if (bytesToSkip <= 0) {
- return;
- }
- long totalSkipped = 0L;
-
- // If we have a FileInputStream, calling skip(1000000) will return 1000000 even if the file is only
- // 3 bytes. As a result, we will skip 1 less than the number requested, and then read the last
- // byte in order to make sure that we've consumed the number of bytes requested. We then check that
- // the final byte, which we read, is not -1.
- final long actualBytesToSkip = bytesToSkip - 1;
- while (totalSkipped < actualBytesToSkip) {
- final long skippedThisIteration = stream.skip(actualBytesToSkip - totalSkipped);
- if (skippedThisIteration == 0) {
- final int nextByte = stream.read();
- if (nextByte == -1) {
- throw new EOFException();
- } else {
- totalSkipped++;
- }
- }
-
- totalSkipped += skippedThisIteration;
- }
-
- final int lastByte = stream.read();
- if (lastByte == -1) {
- throw new EOFException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ZipOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ZipOutputStream.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ZipOutputStream.java
deleted file mode 100644
index f285720..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/ZipOutputStream.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.OutputStream;
-
-/**
- * This class extends the {@link java.util.zip.ZipOutputStream} by providing a
- * constructor that allows the user to specify the compression level. The
- * default compression level is 1, as opposed to Java's default of 5.
- */
-public class ZipOutputStream extends java.util.zip.ZipOutputStream {
-
- public static final int DEFAULT_COMPRESSION_LEVEL = 1;
-
- public ZipOutputStream(final OutputStream out) {
- this(out, DEFAULT_COMPRESSION_LEVEL);
- }
-
- public ZipOutputStream(final OutputStream out, final int compressionLevel) {
- super(out);
- def.setLevel(compressionLevel);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/exception/BytePatternNotFoundException.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/exception/BytePatternNotFoundException.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/exception/BytePatternNotFoundException.java
deleted file mode 100644
index 8935767..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/exception/BytePatternNotFoundException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io.exception;
-
-import java.io.IOException;
-
-public class BytePatternNotFoundException extends IOException {
-
- private static final long serialVersionUID = -4128911284318513973L;
-
- public BytePatternNotFoundException(final String explanation) {
- super(explanation);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/util/NonThreadSafeCircularBuffer.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/util/NonThreadSafeCircularBuffer.java b/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/util/NonThreadSafeCircularBuffer.java
deleted file mode 100644
index 1b87488..0000000
--- a/commons/nifi-stream-utils/src/main/java/org/apache/nifi/io/util/NonThreadSafeCircularBuffer.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io.util;
-
-import java.util.Arrays;
-
-public class NonThreadSafeCircularBuffer {
-
- private final byte[] lookingFor;
- private final int[] buffer;
- private int insertionPointer = 0;
- private int bufferSize = 0;
-
- public NonThreadSafeCircularBuffer(final byte[] lookingFor) {
- this.lookingFor = lookingFor;
- buffer = new int[lookingFor.length];
- Arrays.fill(buffer, -1);
- }
-
- public byte[] getByteArray() {
- return lookingFor;
- }
-
- /**
- * Returns the oldest byte in the buffer
- *
- * @return
- */
- public int getOldestByte() {
- return buffer[insertionPointer];
- }
-
- public boolean isFilled() {
- return bufferSize >= buffer.length;
- }
-
- public boolean addAndCompare(final byte data) {
- buffer[insertionPointer] = data;
- insertionPointer = (insertionPointer + 1) % lookingFor.length;
-
- bufferSize++;
- if (bufferSize < lookingFor.length) {
- return false;
- }
-
- for (int i = 0; i < lookingFor.length; i++) {
- final byte compare = (byte) buffer[(insertionPointer + i) % lookingFor.length];
- if (compare != lookingFor[i]) {
- return false;
- }
- }
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/test/java/org/apache/nifi/io/TestLeakyBucketThrottler.java
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/test/java/org/apache/nifi/io/TestLeakyBucketThrottler.java b/commons/nifi-stream-utils/src/test/java/org/apache/nifi/io/TestLeakyBucketThrottler.java
deleted file mode 100644
index 12e1801..0000000
--- a/commons/nifi-stream-utils/src/test/java/org/apache/nifi/io/TestLeakyBucketThrottler.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-@Ignore("Tests are time-based")
-public class TestLeakyBucketThrottler {
-
- @Test(timeout = 10000)
- public void testOutputStreamInterface() throws IOException {
- // throttle rate at 1 MB/sec
- final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
-
- final byte[] data = new byte[1024 * 1024 * 4];
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final OutputStream throttledOut = throttler.newThrottledOutputStream(baos);
-
- final long start = System.currentTimeMillis();
- throttledOut.write(data);
- throttler.close();
- final long millis = System.currentTimeMillis() - start;
- // should take 4 sec give or take
- assertTrue(millis > 3000);
- assertTrue(millis < 6000);
- }
-
- @Test(timeout = 10000)
- public void testInputStreamInterface() throws IOException {
- // throttle rate at 1 MB/sec
- final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
-
- final byte[] data = new byte[1024 * 1024 * 4];
- final ByteArrayInputStream bais = new ByteArrayInputStream(data);
- final InputStream throttledIn = throttler.newThrottledInputStream(bais);
-
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- final byte[] buffer = new byte[4096];
- final long start = System.currentTimeMillis();
- int len;
- while ((len = throttledIn.read(buffer)) > 0) {
- baos.write(buffer, 0, len);
- }
- throttler.close();
- final long millis = System.currentTimeMillis() - start;
- // should take 4 sec give or take
- assertTrue(millis > 3000);
- assertTrue(millis < 6000);
- baos.close();
- }
-
- @Test(timeout = 10000)
- public void testDirectInterface() throws IOException, InterruptedException {
- // throttle rate at 1 MB/sec
- final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
-
- // create 3 threads, each sending ~2 MB
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final List<Thread> threads = new ArrayList<Thread>();
- for (int i = 0; i < 3; i++) {
- final Thread t = new WriterThread(i, throttler, baos);
- threads.add(t);
- }
-
- final long start = System.currentTimeMillis();
- for (final Thread t : threads) {
- t.start();
- }
-
- for (final Thread t : threads) {
- t.join();
- }
- final long elapsed = System.currentTimeMillis() - start;
-
- throttler.close();
-
- // To send 15 MB, it should have taken at least 5 seconds and no more than 7 seconds, to
- // allow for busy-ness and the fact that we could write a tiny bit more than the limit.
- assertTrue(elapsed > 5000);
- assertTrue(elapsed < 7000);
-
- // ensure bytes were copied out appropriately
- assertEquals(3 * (2 * 1024 * 1024 + 1), baos.getBufferLength());
- assertEquals((byte) 'A', baos.getUnderlyingBuffer()[baos.getBufferLength() - 1]);
- }
-
- private static class WriterThread extends Thread {
-
- private final int idx;
- private final byte[] data = new byte[1024 * 1024 * 2 + 1];
- private final LeakyBucketStreamThrottler throttler;
- private final OutputStream out;
-
- public WriterThread(final int idx, final LeakyBucketStreamThrottler throttler, final OutputStream out) {
- this.idx = idx;
- this.throttler = throttler;
- this.out = out;
- this.data[this.data.length - 1] = (byte) 'A';
- }
-
- @Override
- public void run() {
- long startMillis = System.currentTimeMillis();
- long bytesWritten = 0L;
- try {
- throttler.copy(new ByteArrayInputStream(data), out);
- } catch (IOException e) {
- e.printStackTrace();
- return;
- }
- long now = System.currentTimeMillis();
- long millisElapsed = now - startMillis;
- bytesWritten += data.length;
- float bytesPerSec = (float) bytesWritten / (float) millisElapsed * 1000F;
- System.out.println(idx + " : copied data at a rate of " + bytesPerSec + " bytes/sec");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-stream-utils/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/commons/nifi-stream-utils/src/test/resources/logback-test.xml b/commons/nifi-stream-utils/src/test/resources/logback-test.xml
deleted file mode 100644
index 0f3f60c..0000000
--- a/commons/nifi-stream-utils/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration scan="true" scanPeriod="30 seconds">
- <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
- </encoder>
- </appender>
-
- <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
- <logger name="org.apache.nifi" level="DEBUG"/>
-
- <!-- Logger for managing logging statements for nifi clusters. -->
- <logger name="org.apache.nifi.cluster" level="INFO"/>
-
- <!--
- Logger for logging HTTP requests received by the web server. Setting
- log level to 'debug' activates HTTP request logging.
- -->
- <logger name="org.apache.nifi.server.JettyServer" level="INFO"/>
-
- <!-- Logger for managing logging statements for jetty -->
- <logger name="org.mortbay" level="INFO"/>
-
- <!-- Suppress non-error messages due to excessive logging by class -->
- <logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR"/>
-
- <logger name="org.apache.nifi.processors.standard" level="DEBUG"/>
-
- <root level="INFO">
- <appender-ref ref="CONSOLE"/>
- </root>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/pom.xml b/commons/nifi-utils/pom.xml
index 6a6cee1..c5c2a68 100644
--- a/commons/nifi-utils/pom.xml
+++ b/commons/nifi-utils/pom.xml
@@ -26,4 +26,8 @@
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>NiFi Utils</name>
+ <!--
+ This project intentionally has no additional dependencies beyond that pulled in by the parent. It is a general purpose utility library
+ and should keep its surface/tension minimal.
+ -->
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java b/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
new file mode 100644
index 0000000..24f43ca
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flowfile.attributes;
+
+public enum CoreAttributes implements FlowFileAttributeKey {
+ /**
+ * The flowfile's path indicates the relative directory to which a FlowFile belongs and does not
+ * contain the filename
+ */
+ PATH("path"),
+
+ /**
+ * The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not
+ * contain the filename
+ */
+ ABSOLUTE_PATH("absolute.path"),
+
+ /**
+ * The filename of the FlowFile. The filename should not contain any directory structure.
+ */
+ FILENAME("filename"),
+
+ /**
+ * A unique UUID assigned to this FlowFile
+ */
+ UUID("uuid"),
+
+ /**
+ * A numeric value indicating the FlowFile priority
+ */
+ PRIORITY("priority"),
+
+ /**
+ * The MIME Type of this FlowFile
+ */
+ MIME_TYPE("mime.type"),
+
+ /**
+ * Specifies the reason that a FlowFile is being discarded
+ */
+ DISCARD_REASON("discard.reason"),
+
+ /**
+ * Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile.
+ */
+ ALTERNATE_IDENTIFIER("alternate.identifier");
+
+ private final String key;
+ private CoreAttributes(final String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String key() {
+ return key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java b/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
new file mode 100644
index 0000000..cc6c28e
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flowfile.attributes;
+
+public interface FlowFileAttributeKey {
+ String key();
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/io/CompoundUpdateMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/io/CompoundUpdateMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/io/CompoundUpdateMonitor.java
deleted file mode 100644
index e22032b..0000000
--- a/commons/nifi-utils/src/main/java/org/apache/nifi/io/CompoundUpdateMonitor.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * An {@link UpdateMonitor} that combines multiple <code>UpdateMonitor</code>s
- * such that it will indicate a change in a file only if ALL sub-monitors
- * indicate a change. The sub-monitors will be applied in the order given and if
- * any indicates that the state has not changed, the subsequent sub-monitors may
- * not be given a chance to run
- */
-public class CompoundUpdateMonitor implements UpdateMonitor {
-
- private final List<UpdateMonitor> monitors;
-
- public CompoundUpdateMonitor(final UpdateMonitor first, final UpdateMonitor... others) {
- monitors = new ArrayList<>();
- monitors.add(first);
- for (final UpdateMonitor monitor : others) {
- monitors.add(monitor);
- }
- }
-
- @Override
- public Object getCurrentState(final Path path) throws IOException {
- return new DeferredMonitorAction(monitors, path);
- }
-
- private static class DeferredMonitorAction {
-
- private static final Object NON_COMPUTED_VALUE = new Object();
-
- private final List<UpdateMonitor> monitors;
- private final Path path;
-
- private final Object[] preCalculated;
-
- public DeferredMonitorAction(final List<UpdateMonitor> monitors, final Path path) {
- this.monitors = monitors;
- this.path = path;
- preCalculated = new Object[monitors.size()];
-
- for (int i = 0; i < preCalculated.length; i++) {
- preCalculated[i] = NON_COMPUTED_VALUE;
- }
- }
-
- private Object getCalculatedValue(final int i) throws IOException {
- if (preCalculated[i] == NON_COMPUTED_VALUE) {
- preCalculated[i] = monitors.get(i).getCurrentState(path);
- }
-
- return preCalculated[i];
- }
-
- @Override
- public boolean equals(final Object obj) {
- // must return true unless ALL DeferredMonitorAction's indicate that they are different
- if (obj == null) {
- return false;
- }
-
- if (!(obj instanceof DeferredMonitorAction)) {
- return false;
- }
-
- final DeferredMonitorAction other = (DeferredMonitorAction) obj;
- try {
- // Go through each UpdateMonitor's value and check if the value has changed.
- for (int i = 0; i < preCalculated.length; i++) {
- final Object mine = getCalculatedValue(i);
- final Object theirs = other.getCalculatedValue(i);
-
- if (mine == theirs) {
- // same
- return true;
- }
-
- if (mine == null && theirs == null) {
- // same
- return true;
- }
-
- if (mine.equals(theirs)) {
- return true;
- }
- }
- } catch (final IOException e) {
- return false;
- }
-
- // No DeferredMonitorAction was the same as last time. Therefore, it's not equal
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/io/LastModifiedMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/io/LastModifiedMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/io/LastModifiedMonitor.java
deleted file mode 100644
index f446465..0000000
--- a/commons/nifi-utils/src/main/java/org/apache/nifi/io/LastModifiedMonitor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-public class LastModifiedMonitor implements UpdateMonitor {
-
- @Override
- public Object getCurrentState(final Path path) throws IOException {
- return Files.getLastModifiedTime(path);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/io/MD5SumMonitor.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/io/MD5SumMonitor.java b/commons/nifi-utils/src/main/java/org/apache/nifi/io/MD5SumMonitor.java
deleted file mode 100644
index 1326c2a..0000000
--- a/commons/nifi-utils/src/main/java/org/apache/nifi/io/MD5SumMonitor.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-public class MD5SumMonitor implements UpdateMonitor {
-
- @Override
- public Object getCurrentState(final Path path) throws IOException {
- final MessageDigest digest;
- try {
- digest = MessageDigest.getInstance("MD5");
- } catch (final NoSuchAlgorithmException nsae) {
- throw new AssertionError(nsae);
- }
-
- try (final FileInputStream fis = new FileInputStream(path.toFile())) {
- int len;
- final byte[] buffer = new byte[8192];
- while ((len = fis.read(buffer)) > 0) {
- digest.update(buffer, 0, len);
- }
- }
-
- // Return a ByteBuffer instead of byte[] because we want equals() to do a deep equality
- return ByteBuffer.wrap(digest.digest());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/io/SynchronousFileWatcher.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/io/SynchronousFileWatcher.java b/commons/nifi-utils/src/main/java/org/apache/nifi/io/SynchronousFileWatcher.java
deleted file mode 100644
index 785f1ac..0000000
--- a/commons/nifi-utils/src/main/java/org/apache/nifi/io/SynchronousFileWatcher.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.io;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Allows the user to configure a {@link java.nio.file.Path Path} to watch for
- * modifications and periodically poll to check if the file has been modified
- */
-public class SynchronousFileWatcher {
-
- private final Path path;
- private final long checkUpdateMillis;
- private final UpdateMonitor monitor;
- private final AtomicReference<StateWrapper> lastState;
- private final Lock resourceLock = new ReentrantLock();
-
- public SynchronousFileWatcher(final Path path, final UpdateMonitor monitor) {
- this(path, monitor, 0L);
- }
-
- public SynchronousFileWatcher(final Path path, final UpdateMonitor monitor, final long checkMillis) {
- if (checkMillis < 0) {
- throw new IllegalArgumentException();
- }
-
- this.path = path;
- checkUpdateMillis = checkMillis;
- this.monitor = monitor;
-
- Object currentState;
- try {
- currentState = monitor.getCurrentState(path);
- } catch (final IOException e) {
- currentState = null;
- }
-
- this.lastState = new AtomicReference<>(new StateWrapper(currentState));
- }
-
- /**
- * Checks if the file has been updated according to the configured
- * {@link UpdateMonitor} and resets the state
- *
- * @return
- * @throws IOException
- */
- public boolean checkAndReset() throws IOException {
- if (checkUpdateMillis <= 0) { // if checkUpdateMillis <= 0, always check
- return checkForUpdate();
- } else {
- final StateWrapper stateWrapper = lastState.get();
- if (stateWrapper.getTimestamp() < System.currentTimeMillis() - checkUpdateMillis) {
- return checkForUpdate();
- }
- return false;
- }
- }
-
- private boolean checkForUpdate() throws IOException {
- if (resourceLock.tryLock()) {
- try {
- final StateWrapper wrapper = lastState.get();
- final Object newState = monitor.getCurrentState(path);
- if (newState == null && wrapper.getState() == null) {
- return false;
- }
- if (newState == null || wrapper.getState() == null) {
- lastState.set(new StateWrapper(newState));
- return true;
- }
-
- final boolean unmodified = newState.equals(wrapper.getState());
- if (!unmodified) {
- lastState.set(new StateWrapper(newState));
- }
- return !unmodified;
- } finally {
- resourceLock.unlock();
- }
- } else {
- return false;
- }
- }
-
- private static class StateWrapper {
-
- private final Object state;
- private final long timestamp;
-
- public StateWrapper(final Object state) {
- this.state = state;
- this.timestamp = System.currentTimeMillis();
- }
-
- public Object getState() {
- return state;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
- }
-}
[09/31] incubator-nifi git commit: NIFI-150: Finished 'Anatomy of a
Processor' section
Posted by mc...@apache.org.
NIFI-150: Finished 'Anatomy of a Processor' section
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/62f3354a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/62f3354a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/62f3354a
Branch: refs/heads/NIFI-182
Commit: 62f3354a12d7f5f64c22bc3d11d9c684ad369205
Parents: 782b2c1
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 14:00:49 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 14:00:49 2014 -0500
----------------------------------------------------------------------
.../main/asciidoc/images/processor-anatomy.png | Bin 0 -> 37035 bytes
.../src/main/asciidoc/nifi-user-guide.adoc | 98 ++++++++++++++++++-
2 files changed, 97 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/62f3354a/nifi-docs/src/main/asciidoc/images/processor-anatomy.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/processor-anatomy.png b/nifi-docs/src/main/asciidoc/images/processor-anatomy.png
new file mode 100644
index 0000000..db44d6a
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/processor-anatomy.png differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/62f3354a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
index 62cf8a8..20c5270 100644
--- a/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/nifi-user-guide.adoc
@@ -118,6 +118,7 @@ is a link that will take you back up to that level in the flow.
image::nifi-navigation.png["NiFi Navigation"]
+[[status_bar]]
Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist in the graph in
each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the graph in each state
(Transmitting, Not Transmitting), the number of threads that are currently active in the flow, the amount of data that currently
@@ -628,19 +629,113 @@ or not compression should be used when transmitting data to or from this Port.
+
+
[[monitoring]]
== Monitoring of DataFlow
+NiFi provides a great deal of information about the status of the DataFlow in order to monitor the
+health and status. The Status bar provides information about the overall system health
+(See <<status_bar>> above for more information). Processors, Process Groups, and Remote Process Groups
+provide fine-grained details about their operations. Connections and Process Groups provide information
+about the amount of data in their queues. The Summary Page provides information about all of the components
+on the graph in a tabular format and also provides System Diagnostics information that includes disk usage,
+CPU utilization, and Java Heap and Garbage Collection information. In a clustered environment, this
+information is available per-node or as aggregates across the entire cluster. We will explore each of these
+monitoring artifacts below.
+
+
[[processor_anatomy]]
=== Anatomy of a Processor
+NiFi provides a significant amount of information about each Processor on the canvas. The following diagram
+is the anatomy of a Processor:
+
+image:processor-anatomy.png["Anatomy of a Processor"]
+
+The image outlines the following elements:
+
+- *Processor Type*: NiFi provides several different types of Processors in order to allow for a wide range
+ of tasks to be performed. Each type of Processor is designed to perform one specific task. The Processor
+ type (PutFile, in this example) describes the task that this Processor performs. In this case, the
+ Processor writes a FlowFile to disk - or ``Puts'' a FlowFile to a File.
+
+- *Bulletin Indicator*: When a Processor logs that some event has occurred, it generates a Bulletin to notify
+ those who are monitoring NiFi via the User Interface. The DataFlow Manager is able to configure which
+ bulletins should be displayed in the User Interface by updating the ``Bulletin level'' field in the
+ ``Settings'' tab of the Processor configuration dialog. The default value is `WARN`, which means that only
+ warnings and errors will be displayed in the UI. This icon is not present unless a Bulletin exists for this
+ Processor. When it is present, hovering over the icon with the mouse will provide a tooltip explaining the
+ message provided by the Processor as well as the Bulletin level. If the instance of NiFi is clustered,
+ it will also show the Node that emitted the Bulletin. Bulletins automatically expire after five minutes.
+
+- *Status Indicator*: Shows the current Status of the Processor. The following indicators are possible:
+ ** image:iconRun.png["Running"]
+ *Running*: The Processor is currently running.
+ ** image:iconStop.png["Stopped"]
+ *Stopped*: The Processor is valid and enabled but is not running.
+ ** image:iconAlert.png["Invalid"]
+ *Invalid*: The Processor is enabled but is not currently valid and cannot be started.
+ Hovering over this icon will provide a tooltip indicating why the Processor is not valid.
+ ** image:iconDisable.png["Disabled"]
+ *Disabled*: The Processor is not running and cannot be started until it has been enabled.
+ This status does not indicate whether or not the Processor is valid.
+
+- *Processor Name*: This is the user-defined name of the Processor. By default, the name of the Processor is
+ the same as the Processor Type. In the example, this value is "Copy to /review".
+
+- *Active Tasks*: The number of tasks that this Processor is currently executing. This number is constrained
+ by the ``Concurrent tasks'' setting in the ``Scheduling'' tab of the Processor configuration dialog.
+ Here, we can see that the Processor is currently performing two tasks. If the NiFi instance is clustered,
+ this value represents the number of tasks that are currently executing across all nodes in the cluster.
+
+- *5-Minute Statistics*: The Processor shows several different statistics in tabular form. Each of these
+ statistics represent the amount of work that has been performed in the past five minutes. If the NiFi
+ instance is clustered, these values indicate how much work has been done by all of the Nodes combined
+ in the past five minutes. These metrics are:
+
+ ** *In*: The amount of data that the Processor has pulled from the queues of its incoming Connections.
+ This value is represented as <count> / <size> where <count> is the number of FlowFiles that have been
+ pulled from the queues and <size> is the total size of those FlowFiles' content. In this example,
+ the Processor has pulled 884 FlowFiles from the input queues, for a total of 8.85 megabytes (MB).
+ ** *Read/Write*: The total size of the FlowFile content that the Processor has read from disk and written
+ to disk. This provides valuable information about the I/O performance that this Processor requires.
+ Some Processors may only read the data without writing anything while some will not read the data but
+ will only write data. Others will neither read nor write data, and some Processors will both read
+ and write data. In this example, we see that in the past five minutes, this Processor has read 4.7
+ MB of the FlowFile content and has written 4.7 MB as well. This is what we would expect,
+ since this Processor simply copies the contents of a FlowFile to disk. Note, however, that this is
+ not the same as the amount of data that it pulled from its input queues. This is because some of
+ the files that it pulled from the input queues already exists in the output directory, and the
+ Processor is configured to route FlowFiles to failure when this occurs. Therefore, for those files
+ which already existed in the output directory, no data was read nor written to disk.
+ ** *Out*: The amount of data that the Processor has transferred to its outbound Connections. This does
+ not include FlowFiles that the Processor removes itself, or FlowFiles that are routed to connections
+ that are auto-terminated. Like the ``In'' metric above, this value is represented as <count> / <size>
+ where <count> is the number of FlowFiles that have been transferred to outbound Connections and <size>
+ is the total size of those FlowFiles' content. In this example, all of the Relationships are configured to be
+ auto-terminated, so no FlowFiles are reported as having been transferred Out.
+ ** *Tasks/Time*: The number of times that this Processor has been triggered to run in the past 5 minutes, and
+ the amount of time taken to perform those tasks. The format of the time is <hour>:<minute>:<second>. Note
+ that the amount of time taken can exceed five minutes, because many tasks can be executed in parallel. For
+ instance, if the Processor is scheduled to run with 60 Concurrent tasks, and each of those tasks takes one
+ second to complete, it is possible that all 60 tasks will be completed in a single second. However, in this
+ case we will see the Time metric showing that it took 60 seconds, instead of 1 second. This time can be
+ thought of as ``System Time,'' or said another way, this value is 60 seconds because that's the amount of
+ time it would have taken to perform the action if only a single concurrent task were used.
+
+
+
[[process_group_anatomy]]
=== Anatomy of a Process Group
+
[[remote_group_anatomy]]
=== Anatomy of a Remote Process Group
-=== Status Page
+
+=== Summary Page
+
=== Historical Statics of a Component
@@ -651,6 +746,7 @@ or not compression should be used when transmitting data to or from this Port.
+
== Templates
=== Creating a Template
[26/31] incubator-nifi git commit: NIFI-186 disabled code cahce
options by default
Posted by mc...@apache.org.
NIFI-186 disabled code cahce options by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/965e0ab6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/965e0ab6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/965e0ab6
Branch: refs/heads/NIFI-182
Commit: 965e0ab6130246009e5d8b4c1c6c44e189de1b5d
Parents: 1357a17
Author: joewitt <jo...@apache.org>
Authored: Sat Dec 20 01:06:46 2014 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sat Dec 20 01:06:46 2014 -0500
----------------------------------------------------------------------
.../framework/resources/src/main/resources/conf/bootstrap.conf | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/965e0ab6/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
----------------------------------------------------------------------
diff --git a/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf b/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
index a34fa28..79ece96 100644
--- a/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
+++ b/nar-bundles/framework-bundle/framework/resources/src/main/resources/conf/bootstrap.conf
@@ -29,6 +29,6 @@ java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with
# many classes loaded in the JVM.
-java.arg.7=-XX:ReservedCodeCacheSize=256m
-java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m
-java.arg.9=-XX:+UseCodeCacheFlushing
+#java.arg.7=-XX:ReservedCodeCacheSize=256m
+#java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m
+#java.arg.9=-XX:+UseCodeCacheFlushing
[12/31] incubator-nifi git commit: Merge branch 'NIFI-USER-GUIDE'
into develop
Posted by mc...@apache.org.
Merge branch 'NIFI-USER-GUIDE' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/f048ed71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/f048ed71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/f048ed71
Branch: refs/heads/NIFI-182
Commit: f048ed710f22f443c0bc972bf9618acfe38e5b35
Parents: ea17562 d57b34c
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Dec 19 14:14:44 2014 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Dec 19 14:14:44 2014 -0500
----------------------------------------------------------------------
.../main/webapp/WEB-INF/jsp/documentation.jsp | 1 +
.../src/main/webapp/js/application.js | 5 +
nifi-docs/pom.xml | 85 ++
.../images/add-processor-with-tag-cloud.png | Bin 0 -> 22925 bytes
.../src/main/asciidoc/images/add-processor.png | Bin 0 -> 31524 bytes
.../src/main/asciidoc/images/addConnect.png | Bin 0 -> 1996 bytes
.../src/main/asciidoc/images/comments-tab.png | Bin 0 -> 6431 bytes
.../src/main/asciidoc/images/components.png | Bin 0 -> 8992 bytes
.../asciidoc/images/connection-settings.png | Bin 0 -> 15214 bytes
.../main/asciidoc/images/create-connection.png | Bin 0 -> 9430 bytes
.../asciidoc/images/edit-property-dropdown.png | Bin 0 -> 14291 bytes
.../asciidoc/images/edit-property-textarea.png | Bin 0 -> 17146 bytes
.../src/main/asciidoc/images/iconAlert.png | Bin 0 -> 1396 bytes
.../src/main/asciidoc/images/iconConnection.png | Bin 0 -> 1517 bytes
.../src/main/asciidoc/images/iconDelete.png | Bin 0 -> 670 bytes
.../src/main/asciidoc/images/iconDisable.png | Bin 0 -> 764 bytes
nifi-docs/src/main/asciidoc/images/iconEdit.png | Bin 0 -> 493 bytes
.../src/main/asciidoc/images/iconEnable.png | Bin 0 -> 667 bytes
.../src/main/asciidoc/images/iconFunnel.png | Bin 0 -> 1223 bytes
nifi-docs/src/main/asciidoc/images/iconInfo.png | Bin 0 -> 550 bytes
.../src/main/asciidoc/images/iconInputPort.png | Bin 0 -> 1842 bytes
.../src/main/asciidoc/images/iconLabel.png | Bin 0 -> 838 bytes
.../src/main/asciidoc/images/iconNotSecure.png | Bin 0 -> 137 bytes
.../src/main/asciidoc/images/iconOutputPort.png | Bin 0 -> 1658 bytes
.../main/asciidoc/images/iconProcessGroup.png | Bin 0 -> 1422 bytes
.../src/main/asciidoc/images/iconProcessor.png | Bin 0 -> 1446 bytes
.../asciidoc/images/iconRemoteProcessGroup.png | Bin 0 -> 674 bytes
nifi-docs/src/main/asciidoc/images/iconRun.png | Bin 0 -> 538 bytes
.../src/main/asciidoc/images/iconSecure.png | Bin 0 -> 133 bytes
nifi-docs/src/main/asciidoc/images/iconStop.png | Bin 0 -> 402 bytes
.../src/main/asciidoc/images/iconTemplate.png | Bin 0 -> 970 bytes
.../asciidoc/images/iconTransmissionActive.png | Bin 0 -> 1330 bytes
.../images/iconTransmissionInactive.png | Bin 0 -> 1248 bytes
.../images/instantiate-template-description.png | Bin 0 -> 6508 bytes
.../asciidoc/images/instantiate-template.png | Bin 0 -> 3216 bytes
.../main/asciidoc/images/invalid-processor.png | Bin 0 -> 8187 bytes
nifi-docs/src/main/asciidoc/images/new-flow.png | Bin 0 -> 53989 bytes
.../main/asciidoc/images/nifi-navigation.png | Bin 0 -> 85458 bytes
.../asciidoc/images/nifi-toolbar-components.png | Bin 0 -> 72263 bytes
.../main/asciidoc/images/processor-anatomy.png | Bin 0 -> 37035 bytes
.../images/processor-connection-bubble.png | Bin 0 -> 9936 bytes
.../src/main/asciidoc/images/properties-tab.png | Bin 0 -> 11847 bytes
.../images/remote-group-ports-dialog.png | Bin 0 -> 21662 bytes
.../images/remote-port-connection-status.png | Bin 0 -> 57332 bytes
.../src/main/asciidoc/images/scheduling-tab.png | Bin 0 -> 12288 bytes
.../src/main/asciidoc/images/settings-tab.png | Bin 0 -> 16225 bytes
.../src/main/asciidoc/images/status-bar.png | Bin 0 -> 57171 bytes
.../main/asciidoc/images/valid-processor.png | Bin 0 -> 8090 bytes
.../src/main/asciidoc/nifi-user-guide.adoc | 782 +++++++++++++++++++
pom.xml | 8 +-
50 files changed, 877 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[18/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
index f430199..ab0b2aa 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
@@ -37,9 +37,9 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.AttributeExpression.ResultType;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/HashContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/HashContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/HashContent.java
index d6e454e..bb71589 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/HashContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/HashContent.java
@@ -30,8 +30,8 @@ import java.util.Set;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.NullOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.NullOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenHTTP.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenHTTP.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenHTTP.java
index 23786ab..5e7ce56 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenHTTP.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenHTTP.java
@@ -33,8 +33,8 @@ import javax.ws.rs.Path;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.LeakyBucketStreamThrottler;
-import org.apache.nifi.io.StreamThrottler;
+import org.apache.nifi.stream.io.LeakyBucketStreamThrottler;
+import org.apache.nifi.stream.io.StreamThrottler;
import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
index 9a932f0..b6979c6 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
@@ -46,10 +46,10 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.NonCloseableOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.NonCloseableOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ModifyBytes.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ModifyBytes.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ModifyBytes.java
index 473e3bb..a8b190d 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ModifyBytes.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ModifyBytes.java
@@ -27,7 +27,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
index 9e87c68..14f8a28 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
@@ -47,12 +47,12 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.GZIPOutputStream;
-import org.apache.nifi.io.LeakyBucketStreamThrottler;
-import org.apache.nifi.io.StreamThrottler;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.GZIPOutputStream;
+import org.apache.nifi.stream.io.LeakyBucketStreamThrottler;
+import org.apache.nifi.stream.io.StreamThrottler;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
index 9b99df2..31e5105 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
@@ -18,7 +18,7 @@ package org.apache.nifi.processors.standard;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
index e5e46fd..ae5350b 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
@@ -28,7 +28,7 @@ import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.AttributeValueDecorator;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.annotation.CapabilityDescription;
import org.apache.nifi.processor.annotation.EventDriven;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java
index 0cce0f2..c99935b 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java
@@ -44,7 +44,7 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.expression.AttributeValueDecorator;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteOnContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteOnContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteOnContent.java
index 7446a0d..cb3cff2 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteOnContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteOnContent.java
@@ -32,7 +32,7 @@ import java.util.regex.Pattern;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.expression.AttributeValueDecorator;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.DataUnit;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanAttribute.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanAttribute.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanAttribute.java
index 90f3ea1..df13c66 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanAttribute.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanAttribute.java
@@ -34,8 +34,8 @@ import java.util.regex.Pattern;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.LastModifiedMonitor;
-import org.apache.nifi.io.SynchronousFileWatcher;
+import org.apache.nifi.util.file.monitor.LastModifiedMonitor;
+import org.apache.nifi.util.file.monitor.SynchronousFileWatcher;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanContent.java
index bdf8fa9..9f53469 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/ScanContent.java
@@ -36,9 +36,9 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.LastModifiedMonitor;
-import org.apache.nifi.io.SynchronousFileWatcher;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.util.file.monitor.LastModifiedMonitor;
+import org.apache.nifi.util.file.monitor.SynchronousFileWatcher;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java
index fd8be17..7e67c01 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java
@@ -34,7 +34,7 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
index 6866c35..8520a55 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
@@ -19,10 +19,10 @@ package org.apache.nifi.processors.standard;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.ByteArrayOutputStream;
-import org.apache.nifi.io.ByteCountingInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.ByteCountingInputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java
index 5f0993f..1919dbe 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java
@@ -31,7 +31,7 @@ import javax.xml.parsers.SAXParserFactory;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
index 777479e..7385918 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
@@ -39,7 +39,7 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.AttributeExpression;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index f976bdb..dc6daea 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -33,9 +33,9 @@ import java.util.UUID;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.StreamUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/servlets/ListenHTTPServlet.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/servlets/ListenHTTPServlet.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/servlets/ListenHTTPServlet.java
index d78c139..fb52b80 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/servlets/ListenHTTPServlet.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/servlets/ListenHTTPServlet.java
@@ -44,8 +44,8 @@ import javax.ws.rs.core.MediaType;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.BufferedOutputStream;
-import org.apache.nifi.io.StreamThrottler;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamThrottler;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessSessionFactory;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/util/UDPStreamConsumer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/util/UDPStreamConsumer.java b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/util/UDPStreamConsumer.java
index f0802a8..ad2cca5 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/util/UDPStreamConsumer.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/main/java/org/apache/nifi/processors/standard/util/UDPStreamConsumer.java
@@ -28,7 +28,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.io.nio.BufferPool;
import org.apache.nifi.io.nio.consumer.StreamConsumer;
import org.apache.nifi.logging.ProcessorLog;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java b/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
index 676b586..d3ad30e 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
@@ -16,7 +16,6 @@
*/
package org.apache.nifi.processors.standard;
-import org.apache.nifi.processors.standard.MergeContent;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
@@ -32,7 +31,7 @@ import java.util.Set;
import java.util.zip.ZipInputStream;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayInputStream;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestScanContent.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestScanContent.java b/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestScanContent.java
index 9079f82..8c36845 100644
--- a/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestScanContent.java
+++ b/nar-bundles/standard-bundle/standard-processors/src/test/java/org/apache/nifi/processors/standard/TestScanContent.java
@@ -23,10 +23,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.nifi.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/pom.xml b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/pom.xml
index a251393..22d7c5b 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/pom.xml
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/pom.xml
@@ -42,15 +42,11 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>remote-communications-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
index 4a67864..a0998f6 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
@@ -30,8 +30,8 @@ import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.annotation.OnConfigured;
import org.apache.nifi.distributed.cache.protocol.ProtocolHandshake;
import org.apache.nifi.distributed.cache.protocol.exception.HandshakeException;
-import org.apache.nifi.io.ByteArrayOutputStream;
-import org.apache.nifi.io.DataOutputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.remote.StandardVersionNegotiator;
import org.apache.nifi.remote.VersionNegotiator;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
index c588249..e58e972 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
@@ -30,8 +30,8 @@ import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.annotation.OnConfigured;
import org.apache.nifi.distributed.cache.protocol.ProtocolHandshake;
import org.apache.nifi.distributed.cache.protocol.exception.HandshakeException;
-import org.apache.nifi.io.ByteArrayOutputStream;
-import org.apache.nifi.io.DataOutputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.remote.StandardVersionNegotiator;
import org.apache.nifi.remote.VersionNegotiator;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/SSLCommsSession.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/SSLCommsSession.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/SSLCommsSession.java
index c8be082..9b4b656 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/SSLCommsSession.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/SSLCommsSession.java
@@ -23,8 +23,8 @@ import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannel;
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelInputStream;
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelOutputStream;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
index bbe2917..1f1ff7e 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
@@ -25,8 +25,8 @@ import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.remote.io.InterruptableInputStream;
import org.apache.nifi.remote.io.InterruptableOutputStream;
import org.apache.nifi.remote.io.socket.SocketChannelInputStream;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-protocol/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-protocol/pom.xml b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-protocol/pom.xml
index f636261..99ba060 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-protocol/pom.xml
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-protocol/pom.xml
@@ -32,7 +32,7 @@
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>remote-communications-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/pom.xml b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/pom.xml
index ca96d82..0e14a57 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/pom.xml
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/pom.xml
@@ -37,7 +37,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>remote-communications-utils</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
@@ -45,10 +45,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>ssl-context-service-api</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/AbstractCacheServer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/AbstractCacheServer.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/AbstractCacheServer.java
index 9b4e70e..a9643ab 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/AbstractCacheServer.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/AbstractCacheServer.java
@@ -30,8 +30,8 @@ import javax.net.ssl.SSLContext;
import org.apache.nifi.distributed.cache.protocol.ProtocolHandshake;
import org.apache.nifi.distributed.cache.protocol.exception.HandshakeException;
-import org.apache.nifi.io.BufferedInputStream;
-import org.apache.nifi.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.remote.StandardVersionNegotiator;
import org.apache.nifi.remote.VersionNegotiator;
import org.apache.nifi.remote.io.socket.SocketChannelInputStream;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/SetCacheServer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/SetCacheServer.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/SetCacheServer.java
index 5d2c0f6..d0abe5c 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/SetCacheServer.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/SetCacheServer.java
@@ -29,7 +29,7 @@ import org.apache.nifi.distributed.cache.server.set.PersistentSetCache;
import org.apache.nifi.distributed.cache.server.set.SetCache;
import org.apache.nifi.distributed.cache.server.set.SetCacheResult;
import org.apache.nifi.distributed.cache.server.set.SimpleSetCache;
-import org.apache.nifi.io.DataOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
public class SetCacheServer extends AbstractCacheServer {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/map/MapCacheServer.java
----------------------------------------------------------------------
diff --git a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/map/MapCacheServer.java b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/map/MapCacheServer.java
index 3e8dd0e..e4a600e 100644
--- a/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/map/MapCacheServer.java
+++ b/nar-bundles/standard-services/distributed-cache-services-bundle/distributed-cache-server/src/main/java/org/apache/nifi/distributed/cache/server/map/MapCacheServer.java
@@ -27,7 +27,7 @@ import javax.net.ssl.SSLContext;
import org.apache.nifi.distributed.cache.server.AbstractCacheServer;
import org.apache.nifi.distributed.cache.server.EvictionPolicy;
-import org.apache.nifi.io.DataOutputStream;
+import org.apache.nifi.stream.io.DataOutputStream;
public class MapCacheServer extends AbstractCacheServer {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nar-bundles/update-attribute-bundle/processor/pom.xml
----------------------------------------------------------------------
diff --git a/nar-bundles/update-attribute-bundle/processor/pom.xml b/nar-bundles/update-attribute-bundle/processor/pom.xml
index 5167de6..edef669 100644
--- a/nar-bundles/update-attribute-bundle/processor/pom.xml
+++ b/nar-bundles/update-attribute-bundle/processor/pom.xml
@@ -43,7 +43,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
+ <artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/nifi-mock/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-mock/pom.xml b/nifi-mock/pom.xml
index 2bd53c7..c79e6da 100644
--- a/nifi-mock/pom.xml
+++ b/nifi-mock/pom.xml
@@ -41,10 +41,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>data-provenance-utils</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2ccdaa9..e0d9b47 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,20 +56,20 @@
<maven>${maven.min-version}</maven>
</prerequisites>
<modules>
- <module>commons</module>
<!--
! Would prefer to have the nar-maven-plugin built as part of this parent. However, Maven
! blocks building plugins with extensions in the same reactor as other components which want to use the plugin.
! See MNG-1911 and MNG-5572 for more details.
-->
<!--<module>nar-maven-plugin</module>-->
+ <module>commons</module>
<module>nifi-api</module>
<module>nifi-bootstrap</module>
<module>nifi-mock</module>
<module>nar-bundles</module>
<module>assembly</module>
- <module>nifi-docs</module>
- </modules>
+ <module>nifi-docs</module>
+ </modules>
<scm>
<connection>scm:git:git://git.apache.org/incubator-nifi.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-nifi.git</developerConnection>
@@ -647,11 +647,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-file-utils</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-expression-language</artifactId>
<version>${project.version}</version>
</dependency>
@@ -662,26 +657,11 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-core-flowfile-attributes</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>flowfile-packager</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>naive-search-ring-buffer</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-search-utils</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-socket-utils</artifactId>
<version>${project.version}</version>
</dependency>
@@ -790,11 +770,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-stream-utils</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
<version>${project.version}</version>
</dependency>
@@ -820,11 +795,6 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>remote-communications-utils</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<version>${project.version}</version>
<scope>test</scope>
[22/31] incubator-nifi git commit: NIFI-189 Collapsed several commons
libs into one
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java
new file mode 100644
index 0000000..284cd54
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.InputStream;
+
+/**
+ * This class performs the same function as java.io.ByteArrayInputStream but
+ * does not mark its methods as synchronized
+ */
+public class ByteArrayInputStream extends InputStream {
+
+ /**
+ * An array of bytes that was provided by the creator of the stream.
+ * Elements <code>buf[0]</code> through <code>buf[count-1]</code> are the
+ * only bytes that can ever be read from the stream; element
+ * <code>buf[pos]</code> is the next byte to be read.
+ */
+ protected byte buf[];
+
+ /**
+ * The index of the next character to read from the input stream buffer.
+ * This value should always be nonnegative and not larger than the value of
+ * <code>count</code>. The next byte to be read from the input stream buffer
+ * will be <code>buf[pos]</code>.
+ */
+ protected int pos;
+
+ /**
+ * The currently marked position in the stream. ByteArrayInputStream objects
+ * are marked at position zero by default when constructed. They may be
+ * marked at another position within the buffer by the <code>mark()</code>
+ * method. The current buffer position is set to this point by the
+ * <code>reset()</code> method.
+ * <p>
+ * If no mark has been set, then the value of mark is the offset passed to
+ * the constructor (or 0 if the offset was not supplied).
+ *
+ * @since JDK1.1
+ */
+ protected int mark = 0;
+
+ /**
+ * The index one greater than the last valid character in the input stream
+ * buffer. This value should always be nonnegative and not larger than the
+ * length of <code>buf</code>. It is one greater than the position of the
+ * last byte within <code>buf</code> that can ever be read from the input
+ * stream buffer.
+ */
+ protected int count;
+
+ /**
+ * Creates a <code>ByteArrayInputStream</code> so that it uses
+ * <code>buf</code> as its buffer array. The buffer array is not copied. The
+ * initial value of <code>pos</code> is <code>0</code> and the initial value
+ * of <code>count</code> is the length of <code>buf</code>.
+ *
+ * @param buf the input buffer.
+ */
+ public ByteArrayInputStream(byte buf[]) {
+ this.buf = buf;
+ this.pos = 0;
+ this.count = buf.length;
+ }
+
+ /**
+ * Creates <code>ByteArrayInputStream</code> that uses <code>buf</code> as
+ * its buffer array. The initial value of <code>pos</code> is
+ * <code>offset</code> and the initial value of <code>count</code> is the
+ * minimum of <code>offset+length</code> and <code>buf.length</code>. The
+ * buffer array is not copied. The buffer's mark is set to the specified
+ * offset.
+ *
+ * @param buf the input buffer.
+ * @param offset the offset in the buffer of the first byte to read.
+ * @param length the maximum number of bytes to read from the buffer.
+ */
+ public ByteArrayInputStream(byte buf[], int offset, int length) {
+ this.buf = buf;
+ this.pos = offset;
+ this.count = Math.min(offset + length, buf.length);
+ this.mark = offset;
+ }
+
+ /**
+ * Reads the next byte of data from this input stream. The value byte is
+ * returned as an <code>int</code> in the range <code>0</code> to
+ * <code>255</code>. If no byte is available because the end of the stream
+ * has been reached, the value <code>-1</code> is returned.
+ * <p>
+ * This <code>read</code> method cannot block.
+ *
+ * @return the next byte of data, or <code>-1</code> if the end of the
+ * stream has been reached.
+ */
+ @Override
+ public int read() {
+ return (pos < count) ? (buf[pos++] & 0xff) : -1;
+ }
+
+ /**
+ * Reads up to <code>len</code> bytes of data into an array of bytes from
+ * this input stream. If <code>pos</code> equals <code>count</code>, then
+ * <code>-1</code> is returned to indicate end of file. Otherwise, the
+ * number <code>k</code> of bytes read is equal to the smaller of
+ * <code>len</code> and <code>count-pos</code>. If <code>k</code> is
+ * positive, then bytes <code>buf[pos]</code> through
+ * <code>buf[pos+k-1]</code> are copied into <code>b[off]</code> through
+ * <code>b[off+k-1]</code> in the manner performed by
+ * <code>System.arraycopy</code>. The value <code>k</code> is added into
+ * <code>pos</code> and <code>k</code> is returned.
+ * <p>
+ * This <code>read</code> method cannot block.
+ *
+ * @param b the buffer into which the data is read.
+ * @param off the start offset in the destination array <code>b</code>
+ * @param len the maximum number of bytes read.
+ * @return the total number of bytes read into the buffer, or
+ * <code>-1</code> if there is no more data because the end of the stream
+ * has been reached.
+ * @exception NullPointerException If <code>b</code> is <code>null</code>.
+ * @exception IndexOutOfBoundsException If <code>off</code> is negative,
+ * <code>len</code> is negative, or <code>len</code> is greater than
+ * <code>b.length - off</code>
+ */
+ @Override
+ public int read(byte b[], int off, int len) {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ if (pos >= count) {
+ return -1;
+ }
+
+ int avail = count - pos;
+ if (len > avail) {
+ len = avail;
+ }
+ if (len <= 0) {
+ return 0;
+ }
+ System.arraycopy(buf, pos, b, off, len);
+ pos += len;
+ return len;
+ }
+
+ /**
+ * Skips <code>n</code> bytes of input from this input stream. Fewer bytes
+ * might be skipped if the end of the input stream is reached. The actual
+ * number <code>k</code> of bytes to be skipped is equal to the smaller of
+ * <code>n</code> and <code>count-pos</code>. The value <code>k</code> is
+ * added into <code>pos</code> and <code>k</code> is returned.
+ *
+ * @param n the number of bytes to be skipped.
+ * @return the actual number of bytes skipped.
+ */
+ @Override
+ public long skip(long n) {
+ long k = count - pos;
+ if (n < k) {
+ k = n < 0 ? 0 : n;
+ }
+
+ pos += k;
+ return k;
+ }
+
+ /**
+ * Returns the number of remaining bytes that can be read (or skipped over)
+ * from this input stream.
+ * <p>
+ * The value returned is <code>count - pos</code>, which is the number
+ * of bytes remaining to be read from the input buffer.
+ *
+ * @return the number of remaining bytes that can be read (or skipped over)
+ * from this input stream without blocking.
+ */
+ @Override
+ public int available() {
+ return count - pos;
+ }
+
+ /**
+ * Tests if this <code>InputStream</code> supports mark/reset. The
+ * <code>markSupported</code> method of <code>ByteArrayInputStream</code>
+ * always returns <code>true</code>.
+ *
+ * @since JDK1.1
+ */
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ /**
+ * Set the current marked position in the stream. ByteArrayInputStream
+ * objects are marked at position zero by default when constructed. They may
+ * be marked at another position within the buffer by this method.
+ * <p>
+ * If no mark has been set, then the value of the mark is the offset passed
+ * to the constructor (or 0 if the offset was not supplied).
+ *
+ * <p>
+ * Note: The <code>readAheadLimit</code> for this class has no meaning.
+ *
+ * @since JDK1.1
+ */
+ @Override
+ public void mark(int readAheadLimit) {
+ mark = pos;
+ }
+
+ /**
+ * Resets the buffer to the marked position. The marked position is 0 unless
+ * another position was marked or an offset was specified in the
+ * constructor.
+ */
+ @Override
+ public void reset() {
+ pos = mark;
+ }
+
+ /**
+ * Closing a <tt>ByteArrayInputStream</tt> has no effect. The methods in
+ * this class can be called after the stream has been closed without
+ * generating an <tt>IOException</tt>.
+ * <p>
+ */
+ @Override
+ public void close() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java
new file mode 100644
index 0000000..459563b
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
+/**
+ * This class provides a more efficient implementation of the
+ * java.io.ByteArrayOutputStream. The efficiency is gained in two ways:
+ * <ul>
+ * <li>The write methods are not synchronized</li>
+ * <li>The class provides {@link #getUnderlyingBuffer()} and
+ * {@link #getBufferLength()}, which can be used to access the underlying byte
+ * array directly, rather than the System.arraycopy that {@link #toByteArray()}
+ * uses
+ * </ul>
+ *
+ */
+public class ByteArrayOutputStream extends OutputStream {
+
+ /**
+ * The buffer where data is stored.
+ */
+ protected byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer.
+ */
+ protected int count;
+
+ /**
+ * Creates a new byte array output stream. The buffer capacity is initially
+ * 32 bytes, though its size increases if necessary.
+ */
+ public ByteArrayOutputStream() {
+ this(32);
+ }
+
+ /**
+ * Creates a new byte array output stream, with a buffer capacity of the
+ * specified size, in bytes.
+ *
+ * @param size the initial size.
+ * @exception IllegalArgumentException if size is negative.
+ */
+ public ByteArrayOutputStream(int size) {
+ if (size < 0) {
+ throw new IllegalArgumentException("Negative initial size: "
+ + size);
+ }
+ buf = new byte[size];
+ }
+
+ /**
+ * Increases the capacity if necessary to ensure that it can hold at least
+ * the number of elements specified by the minimum capacity argument.
+ *
+ * @param minCapacity the desired minimum capacity
+ * @throws OutOfMemoryError if {@code minCapacity < 0}. This is interpreted
+ * as a request for the unsatisfiably large capacity
+ * {@code (long) Integer.MAX_VALUE + (minCapacity - Integer.MAX_VALUE)}.
+ */
+ private void ensureCapacity(int minCapacity) {
+ // overflow-conscious code
+ if (minCapacity - buf.length > 0) {
+ grow(minCapacity);
+ }
+ }
+
+ /**
+ * Increases the capacity to ensure that it can hold at least the number of
+ * elements specified by the minimum capacity argument.
+ *
+ * @param minCapacity the desired minimum capacity
+ */
+ private void grow(int minCapacity) {
+ // overflow-conscious code
+ int oldCapacity = buf.length;
+ int newCapacity = oldCapacity << 1;
+ if (newCapacity - minCapacity < 0) {
+ newCapacity = minCapacity;
+ }
+ if (newCapacity < 0) {
+ if (minCapacity < 0) // overflow
+ {
+ throw new OutOfMemoryError();
+ }
+ newCapacity = Integer.MAX_VALUE;
+ }
+ buf = Arrays.copyOf(buf, newCapacity);
+ }
+
+ /**
+ * Writes the specified byte to this byte array output stream.
+ *
+ * @param b the byte to be written.
+ */
+ @Override
+ public void write(int b) {
+ ensureCapacity(count + 1);
+ buf[count] = (byte) b;
+ count += 1;
+ }
+
+ /**
+ * Writes <code>len</code> bytes from the specified byte array starting at
+ * offset <code>off</code> to this byte array output stream.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ */
+ @Override
+ public void write(byte b[], int off, int len) {
+ if ((off < 0) || (off > b.length) || (len < 0)
+ || ((off + len) - b.length > 0)) {
+ throw new IndexOutOfBoundsException();
+ }
+ ensureCapacity(count + len);
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ /**
+ * Writes the complete contents of this byte array output stream to the
+ * specified output stream argument, as if by calling the output stream's
+ * write method using <code>out.write(buf, 0, count)</code>.
+ *
+ * @param out the output stream to which to write the data.
+ * @exception IOException if an I/O error occurs.
+ */
+ public void writeTo(OutputStream out) throws IOException {
+ out.write(buf, 0, count);
+ }
+
+ /**
+ * Resets the <code>count</code> field of this byte array output stream to
+ * zero, so that all currently accumulated output in the output stream is
+ * discarded. The output stream can be used again, reusing the already
+ * allocated buffer space.
+ *
+ * @see java.io.ByteArrayInputStream#count
+ */
+ public void reset() {
+ count = 0;
+ }
+
+ /**
+ * Creates a newly allocated byte array. Its size is the current size of
+ * this output stream and the valid contents of the buffer have been copied
+ * into it.
+ *
+ * @return the current contents of this output stream, as a byte array.
+ * @see java.io.ByteArrayOutputStream#size()
+ */
+ public byte toByteArray ()
+ [] {
+ return Arrays.copyOf(buf, count);
+ }
+
+ /**
+ * Returns the current size of the buffer.
+ *
+ * @return the value of the <code>count</code> field, which is the number of
+ * valid bytes in this output stream.
+ * @see java.io.ByteArrayOutputStream#count
+ */
+ public int size() {
+ return count;
+ }
+
+ /**
+ * Converts the buffer's contents into a string decoding bytes using the
+ * platform's default character set. The length of the new <tt>String</tt>
+ * is a function of the character set, and hence may not be equal to the
+ * size of the buffer.
+ *
+ * <p>
+ * This method always replaces malformed-input and unmappable-character
+ * sequences with the default replacement string for the platform's default
+ * character set. The {@linkplain java.nio.charset.CharsetDecoder} class
+ * should be used when more control over the decoding process is required.
+ *
+ * @return String decoded from the buffer's contents.
+ * @since JDK1.1
+ */
+ @Override
+ public String toString() {
+ return new String(buf, 0, count);
+ }
+
+ /**
+ * Converts the buffer's contents into a string by decoding the bytes using
+ * the specified {@link java.nio.charset.Charset charsetName}. The length of
+ * the new <tt>String</tt> is a function of the charset, and hence may not
+ * be equal to the length of the byte array.
+ *
+ * <p>
+ * This method always replaces malformed-input and unmappable-character
+ * sequences with this charset's default replacement string. The {@link
+ * java.nio.charset.CharsetDecoder} class should be used when more control
+ * over the decoding process is required.
+ *
+ * @param charsetName the name of a supported
+ * {@linkplain java.nio.charset.Charset <code>charset</code>}
+ * @return String decoded from the buffer's contents.
+ * @exception UnsupportedEncodingException If the named charset is not
+ * supported
+ * @since JDK1.1
+ */
+ public String toString(String charsetName) throws UnsupportedEncodingException {
+ return new String(buf, 0, count, charsetName);
+ }
+
+ /**
+ * Closing a <tt>ByteArrayOutputStream</tt> has no effect. The methods in
+ * this class can be called after the stream has been closed without
+ * generating an <tt>IOException</tt>.
+ * <p>
+ *
+ */
+ @Override
+ public void close() {
+ }
+
+ public byte[] getUnderlyingBuffer() {
+ return buf;
+ }
+
+ public int getBufferLength() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
new file mode 100644
index 0000000..8294af3
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class ByteCountingInputStream extends InputStream {
+
+ private final InputStream in;
+ private long bytesRead = 0L;
+ private long bytesSkipped = 0L;
+
+ private long bytesSinceMark = 0L;
+
+ public ByteCountingInputStream(final InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ final int fromSuper = in.read();
+ if (fromSuper >= 0) {
+ bytesRead++;
+ bytesSinceMark++;
+ }
+ return fromSuper;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ final int fromSuper = in.read(b, off, len);
+ if (fromSuper >= 0) {
+ bytesRead += fromSuper;
+ bytesSinceMark += fromSuper;
+ }
+
+ return fromSuper;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ final long skipped = in.skip(n);
+ if (skipped >= 0) {
+ bytesSkipped += skipped;
+ bytesSinceMark += skipped;
+ }
+ return skipped;
+ }
+
+ public long getBytesRead() {
+ return bytesRead;
+ }
+
+ public long getBytesSkipped() {
+ return bytesSkipped;
+ }
+
+ public long getBytesConsumed() {
+ return getBytesRead() + getBytesSkipped();
+ }
+
+ @Override
+ public void mark(final int readlimit) {
+ in.mark(readlimit);
+
+ bytesSinceMark = 0L;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ in.reset();
+ bytesRead -= bytesSinceMark;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
new file mode 100644
index 0000000..d8e1a42
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class ByteCountingOutputStream extends OutputStream {
+
+ private final OutputStream out;
+ private long bytesWritten = 0L;
+
+ public ByteCountingOutputStream(final OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ bytesWritten++;
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ ;
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ bytesWritten += len;
+ }
+
+ public long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java
new file mode 100644
index 0000000..1dd90f5
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.DataOutput;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UTFDataFormatException;
+
+/**
+ * This class is different from java.io.DataOutputStream in that it does
+ * synchronize on its methods.
+ */
+public class DataOutputStream extends FilterOutputStream implements DataOutput {
+
+ /**
+ * The number of bytes written to the data output stream so far. If this
+ * counter overflows, it will be wrapped to Integer.MAX_VALUE.
+ */
+ protected int written;
+
+ /**
+ * bytearr is initialized on demand by writeUTF
+ */
+ private byte[] bytearr = null;
+
+ /**
+ * Creates a new data output stream to write data to the specified
+ * underlying output stream. The counter <code>written</code> is set to
+ * zero.
+ *
+ * @param out the underlying output stream, to be saved for later use.
+ * @see java.io.FilterOutputStream#out
+ */
+ public DataOutputStream(OutputStream out) {
+ super(out);
+ }
+
+ /**
+ * Increases the written counter by the specified value until it reaches
+ * Integer.MAX_VALUE.
+ */
+ private void incCount(int value) {
+ int temp = written + value;
+ if (temp < 0) {
+ temp = Integer.MAX_VALUE;
+ }
+ written = temp;
+ }
+
+ /**
+ * Writes the specified byte (the low eight bits of the argument
+ * <code>b</code>) to the underlying output stream. If no exception is
+ * thrown, the counter <code>written</code> is incremented by
+ * <code>1</code>.
+ * <p>
+ * Implements the <code>write</code> method of <code>OutputStream</code>.
+ *
+ * @param b the <code>byte</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ incCount(1);
+ }
+
+ /**
+ * Writes <code>len</code> bytes from the specified byte array starting at
+ * offset <code>off</code> to the underlying output stream. If no exception
+ * is thrown, the counter <code>written</code> is incremented by
+ * <code>len</code>.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ out.write(b, off, len);
+ incCount(len);
+ }
+
+ /**
+ * Flushes this data output stream. This forces any buffered output bytes to
+ * be written out to the stream.
+ * <p>
+ * The <code>flush</code> method of <code>DataOutputStream</code> calls the
+ * <code>flush</code> method of its underlying output stream.
+ *
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ * @see java.io.OutputStream#flush()
+ */
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ /**
+ * Writes a <code>boolean</code> to the underlying output stream as a 1-byte
+ * value. The value <code>true</code> is written out as the value
+ * <code>(byte)1</code>; the value <code>false</code> is written out as the
+ * value <code>(byte)0</code>. If no exception is thrown, the counter
+ * <code>written</code> is incremented by <code>1</code>.
+ *
+ * @param v a <code>boolean</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeBoolean(boolean v) throws IOException {
+ out.write(v ? 1 : 0);
+ incCount(1);
+ }
+
+ /**
+ * Writes out a <code>byte</code> to the underlying output stream as a
+ * 1-byte value. If no exception is thrown, the counter <code>written</code>
+ * is incremented by <code>1</code>.
+ *
+ * @param v a <code>byte</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeByte(int v) throws IOException {
+ out.write(v);
+ incCount(1);
+ }
+
+ /**
+ * Writes a <code>short</code> to the underlying output stream as two bytes,
+ * high byte first. If no exception is thrown, the counter
+ * <code>written</code> is incremented by <code>2</code>.
+ *
+ * @param v a <code>short</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeShort(int v) throws IOException {
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ incCount(2);
+ }
+
+ /**
+ * Writes a <code>char</code> to the underlying output stream as a 2-byte
+ * value, high byte first. If no exception is thrown, the counter
+ * <code>written</code> is incremented by <code>2</code>.
+ *
+ * @param v a <code>char</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeChar(int v) throws IOException {
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ incCount(2);
+ }
+
+ /**
+ * Writes an <code>int</code> to the underlying output stream as four bytes,
+ * high byte first. If no exception is thrown, the counter
+ * <code>written</code> is incremented by <code>4</code>.
+ *
+ * @param v an <code>int</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeInt(int v) throws IOException {
+ out.write((v >>> 24) & 0xFF);
+ out.write((v >>> 16) & 0xFF);
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ incCount(4);
+ }
+
+ private final byte writeBuffer[] = new byte[8];
+
+ /**
+ * Writes a <code>long</code> to the underlying output stream as eight
+ * bytes, high byte first. In no exception is thrown, the counter
+ * <code>written</code> is incremented by <code>8</code>.
+ *
+ * @param v a <code>long</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeLong(long v) throws IOException {
+ writeBuffer[0] = (byte) (v >>> 56);
+ writeBuffer[1] = (byte) (v >>> 48);
+ writeBuffer[2] = (byte) (v >>> 40);
+ writeBuffer[3] = (byte) (v >>> 32);
+ writeBuffer[4] = (byte) (v >>> 24);
+ writeBuffer[5] = (byte) (v >>> 16);
+ writeBuffer[6] = (byte) (v >>> 8);
+ writeBuffer[7] = (byte) (v);
+ out.write(writeBuffer, 0, 8);
+ incCount(8);
+ }
+
+ /**
+ * Converts the float argument to an <code>int</code> using the
+ * <code>floatToIntBits</code> method in class <code>Float</code>, and then
+ * writes that <code>int</code> value to the underlying output stream as a
+ * 4-byte quantity, high byte first. If no exception is thrown, the counter
+ * <code>written</code> is incremented by <code>4</code>.
+ *
+ * @param v a <code>float</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ * @see java.lang.Float#floatToIntBits(float)
+ */
+ @Override
+ public final void writeFloat(float v) throws IOException {
+ writeInt(Float.floatToIntBits(v));
+ }
+
+ /**
+ * Converts the double argument to a <code>long</code> using the
+ * <code>doubleToLongBits</code> method in class <code>Double</code>, and
+ * then writes that <code>long</code> value to the underlying output stream
+ * as an 8-byte quantity, high byte first. If no exception is thrown, the
+ * counter <code>written</code> is incremented by <code>8</code>.
+ *
+ * @param v a <code>double</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ * @see java.lang.Double#doubleToLongBits(double)
+ */
+ @Override
+ public final void writeDouble(double v) throws IOException {
+ writeLong(Double.doubleToLongBits(v));
+ }
+
+ /**
+ * Writes out the string to the underlying output stream as a sequence of
+ * bytes. Each character in the string is written out, in sequence, by
+ * discarding its high eight bits. If no exception is thrown, the counter
+ * <code>written</code> is incremented by the length of <code>s</code>.
+ *
+ * @param s a string of bytes to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeBytes(String s) throws IOException {
+ int len = s.length();
+ for (int i = 0; i < len; i++) {
+ out.write((byte) s.charAt(i));
+ }
+ incCount(len);
+ }
+
+ /**
+ * Writes a string to the underlying output stream as a sequence of
+ * characters. Each character is written to the data output stream as if by
+ * the <code>writeChar</code> method. If no exception is thrown, the counter
+ * <code>written</code> is incremented by twice the length of
+ * <code>s</code>.
+ *
+ * @param s a <code>String</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.DataOutputStream#writeChar(int)
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeChars(String s) throws IOException {
+ int len = s.length();
+ for (int i = 0; i < len; i++) {
+ int v = s.charAt(i);
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ }
+ incCount(len * 2);
+ }
+
+ /**
+ * Writes a string to the underlying output stream using
+ * <a href="DataInput.html#modified-utf-8">modified UTF-8</a>
+ * encoding in a machine-independent manner.
+ * <p>
+ * First, two bytes are written to the output stream as if by the
+ * <code>writeShort</code> method giving the number of bytes to follow. This
+ * value is the number of bytes actually written out, not the length of the
+ * string. Following the length, each character of the string is output, in
+ * sequence, using the modified UTF-8 encoding for the character. If no
+ * exception is thrown, the counter <code>written</code> is incremented by
+ * the total number of bytes written to the output stream. This will be at
+ * least two plus the length of <code>str</code>, and at most two plus
+ * thrice the length of <code>str</code>.
+ *
+ * @param str a string to be written.
+ * @exception IOException if an I/O error occurs.
+ */
+ @Override
+ public final void writeUTF(String str) throws IOException {
+ writeUTF(str, this);
+ }
+
+ /**
+ * Writes a string to the specified DataOutput using
+ * <a href="DataInput.html#modified-utf-8">modified UTF-8</a>
+ * encoding in a machine-independent manner.
+ * <p>
+ * First, two bytes are written to out as if by the <code>writeShort</code>
+ * method giving the number of bytes to follow. This value is the number of
+ * bytes actually written out, not the length of the string. Following the
+ * length, each character of the string is output, in sequence, using the
+ * modified UTF-8 encoding for the character. If no exception is thrown, the
+ * counter <code>written</code> is incremented by the total number of bytes
+ * written to the output stream. This will be at least two plus the length
+ * of <code>str</code>, and at most two plus thrice the length of
+ * <code>str</code>.
+ *
+ * @param str a string to be written.
+ * @param out destination to write to
+ * @return The number of bytes written out.
+ * @exception IOException if an I/O error occurs.
+ */
+ static int writeUTF(String str, DataOutput out) throws IOException {
+ int strlen = str.length();
+ int utflen = 0;
+ int c, count = 0;
+
+ /* use charAt instead of copying String to char array */
+ for (int i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ utflen++;
+ } else if (c > 0x07FF) {
+ utflen += 3;
+ } else {
+ utflen += 2;
+ }
+ }
+
+ if (utflen > 65535) {
+ throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes");
+ }
+
+ byte[] bytearr = null;
+ if (out instanceof DataOutputStream) {
+ DataOutputStream dos = (DataOutputStream) out;
+ if (dos.bytearr == null || (dos.bytearr.length < (utflen + 2))) {
+ dos.bytearr = new byte[(utflen * 2) + 2];
+ }
+ bytearr = dos.bytearr;
+ } else {
+ bytearr = new byte[utflen + 2];
+ }
+
+ bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF);
+ bytearr[count++] = (byte) ((utflen) & 0xFF);
+
+ int i = 0;
+ for (i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if (!((c >= 0x0001) && (c <= 0x007F))) {
+ break;
+ }
+ bytearr[count++] = (byte) c;
+ }
+
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ bytearr[count++] = (byte) c;
+
+ } else if (c > 0x07FF) {
+ bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
+ bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
+ } else {
+ bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
+ bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
+ }
+ }
+ out.write(bytearr, 0, utflen + 2);
+ return utflen + 2;
+ }
+
+ /**
+ * Returns the current value of the counter <code>written</code>, the number
+ * of bytes written to this data output stream so far. If the counter
+ * overflows, it will be wrapped to Integer.MAX_VALUE.
+ *
+ * @return the value of the <code>written</code> field.
+ * @see java.io.DataOutputStream#written
+ */
+ public final int size() {
+ return written;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java
new file mode 100644
index 0000000..2864bbb
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * <p>
+ * This class extends the {@link java.util.zip.GZIPOutputStream} by allowing the
+ * constructor to provide a compression level, and uses a default value of 1,
+ * rather than 5.
+ * </p>
+ */
+public class GZIPOutputStream extends java.util.zip.GZIPOutputStream {
+
+ public static final int DEFAULT_COMPRESSION_LEVEL = 1;
+
+ public GZIPOutputStream(final OutputStream out) throws IOException {
+ this(out, DEFAULT_COMPRESSION_LEVEL);
+ }
+
+ public GZIPOutputStream(final OutputStream out, final int compressionLevel) throws IOException {
+ super(out);
+ def.setLevel(compressionLevel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java
new file mode 100644
index 0000000..bffbe26
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class LeakyBucketStreamThrottler implements StreamThrottler {
+
+ private final int maxBytesPerSecond;
+ private final BlockingQueue<Request> requestQueue = new LinkedBlockingQueue<Request>();
+ private final ScheduledExecutorService executorService;
+ private final AtomicBoolean shutdown = new AtomicBoolean(false);
+
+ public LeakyBucketStreamThrottler(final int maxBytesPerSecond) {
+ this.maxBytesPerSecond = maxBytesPerSecond;
+
+ executorService = Executors.newSingleThreadScheduledExecutor();
+ final Runnable task = new Drain();
+ executorService.scheduleAtFixedRate(task, 0, 1000, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void close() {
+ this.shutdown.set(true);
+
+ executorService.shutdown();
+ try {
+ // Should not take more than 2 seconds because we run every second. If it takes more than
+ // 2 seconds, it is because the Runnable thread is blocking on a write; in this case,
+ // we will just ignore it and return
+ executorService.awaitTermination(2, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ }
+ }
+
+ @Override
+ public OutputStream newThrottledOutputStream(final OutputStream toWrap) {
+ return new OutputStream() {
+ @Override
+ public void write(final int b) throws IOException {
+ write(new byte[]{(byte) b}, 0, 1);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ final InputStream in = new ByteArrayInputStream(b, off, len);
+ LeakyBucketStreamThrottler.this.copy(in, toWrap);
+ }
+
+ @Override
+ public void close() throws IOException {
+ toWrap.close();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ toWrap.flush();
+ }
+ };
+ }
+
+ @Override
+ public InputStream newThrottledInputStream(final InputStream toWrap) {
+ return new InputStream() {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Override
+ public int read() throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream(1);
+ LeakyBucketStreamThrottler.this.copy(toWrap, baos, 1L);
+ if (baos.getBufferLength() < 1) {
+ return -1;
+ }
+
+ return baos.getUnderlyingBuffer()[0] & 0xFF;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ if(b.length == 0){
+ return 0;
+ }
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ baos.reset();
+ final int copied = (int) LeakyBucketStreamThrottler.this.copy(toWrap, baos, len);
+ if (copied == 0) {
+ return -1;
+ }
+ System.arraycopy(baos.getUnderlyingBuffer(), 0, b, off, copied);
+ return copied;
+ }
+
+ @Override
+ public void close() throws IOException {
+ toWrap.close();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return toWrap.available();
+ }
+ };
+ }
+
+ @Override
+ public long copy(final InputStream in, final OutputStream out) throws IOException {
+ return copy(in, out, -1);
+ }
+
+ @Override
+ public long copy(final InputStream in, final OutputStream out, final long maxBytes) throws IOException {
+ long totalBytesCopied = 0;
+ boolean finished = false;
+ while (!finished) {
+ final long requestMax = (maxBytes < 0) ? Long.MAX_VALUE : maxBytes - totalBytesCopied;
+ final Request request = new Request(in, out, requestMax);
+ boolean transferred = false;
+ while (!transferred) {
+ if (shutdown.get()) {
+ throw new IOException("Throttler shutdown");
+ }
+
+ try {
+ transferred = requestQueue.offer(request, 1000, TimeUnit.MILLISECONDS);
+ } catch (final InterruptedException e) {
+ throw new IOException("Interrupted", e);
+ }
+ }
+
+ final BlockingQueue<Response> responseQueue = request.getResponseQueue();
+ Response response = null;
+ while (response == null) {
+ try {
+ if (shutdown.get()) {
+ throw new IOException("Throttler shutdown");
+ }
+ response = responseQueue.poll(1000L, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted", e);
+ }
+ }
+
+ if (!response.isSuccess()) {
+ throw response.getError();
+ }
+
+ totalBytesCopied += response.getBytesCopied();
+ finished = (response.getBytesCopied() == 0) || (totalBytesCopied >= maxBytes && maxBytes > 0);
+ }
+
+ return totalBytesCopied;
+ }
+
+ /**
+ * This class is responsible for draining water from the leaky bucket. I.e.,
+ * it actually moves the data
+ */
+ private class Drain implements Runnable {
+
+ private final byte[] buffer;
+
+ public Drain() {
+ final int bufferSize = Math.min(4096, maxBytesPerSecond);
+ buffer = new byte[bufferSize];
+ }
+
+ @Override
+ public void run() {
+ final long start = System.currentTimeMillis();
+
+ int bytesTransferred = 0;
+ while (bytesTransferred < maxBytesPerSecond) {
+ final long maxMillisToWait = 1000 - (System.currentTimeMillis() - start);
+ if (maxMillisToWait < 1) {
+ return;
+ }
+
+ try {
+ final Request request = requestQueue.poll(maxMillisToWait, TimeUnit.MILLISECONDS);
+ if (request == null) {
+ return;
+ }
+
+ final BlockingQueue<Response> responseQueue = request.getResponseQueue();
+
+ final OutputStream out = request.getOutputStream();
+ final InputStream in = request.getInputStream();
+
+ try {
+ final long requestMax = request.getMaxBytesToCopy();
+ long maxBytesToTransfer;
+ if (requestMax < 0) {
+ maxBytesToTransfer = Math.min(buffer.length, maxBytesPerSecond - bytesTransferred);
+ } else {
+ maxBytesToTransfer = Math.min(requestMax,
+ Math.min(buffer.length, maxBytesPerSecond - bytesTransferred));
+ }
+ maxBytesToTransfer = Math.max(1L, maxBytesToTransfer);
+
+ final int bytesCopied = fillBuffer(in, maxBytesToTransfer);
+ out.write(buffer, 0, bytesCopied);
+
+ final Response response = new Response(true, bytesCopied);
+ responseQueue.put(response);
+ bytesTransferred += bytesCopied;
+ } catch (final IOException e) {
+ final Response response = new Response(e);
+ responseQueue.put(response);
+ }
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+
+ private int fillBuffer(final InputStream in, final long maxBytes) throws IOException {
+ int bytesRead = 0;
+ int len;
+ while (bytesRead < maxBytes && (len = in.read(buffer, bytesRead, (int) Math.min(maxBytes - bytesRead, buffer.length - bytesRead))) > 0) {
+ bytesRead += len;
+ }
+
+ return bytesRead;
+ }
+ }
+
+ private static class Response {
+
+ private final boolean success;
+ private final IOException error;
+ private final int bytesCopied;
+
+ public Response(final boolean success, final int bytesCopied) {
+ this.success = success;
+ this.bytesCopied = bytesCopied;
+ this.error = null;
+ }
+
+ public Response(final IOException error) {
+ this.success = false;
+ this.error = error;
+ this.bytesCopied = -1;
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public IOException getError() {
+ return error;
+ }
+
+ public int getBytesCopied() {
+ return bytesCopied;
+ }
+ }
+
+ private static class Request {
+
+ private final OutputStream out;
+ private final InputStream in;
+ private final long maxBytesToCopy;
+ private final BlockingQueue<Response> responseQueue;
+
+ public Request(final InputStream in, final OutputStream out, final long maxBytesToCopy) {
+ this.out = out;
+ this.in = in;
+ this.maxBytesToCopy = maxBytesToCopy;
+ this.responseQueue = new LinkedBlockingQueue<Response>(1);
+ }
+
+ public BlockingQueue<Response> getResponseQueue() {
+ return this.responseQueue;
+ }
+
+ public OutputStream getOutputStream() {
+ return out;
+ }
+
+ public InputStream getInputStream() {
+ return in;
+ }
+
+ public long getMaxBytesToCopy() {
+ return maxBytesToCopy;
+ }
+
+ @Override
+ public String toString() {
+ return "Request[maxBytes=" + maxBytesToCopy + "]";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java
new file mode 100644
index 0000000..0e75a22
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Wraps and InputStream so that the underlying InputStream cannot be closed.
+ * This is used so that the InputStream can be wrapped with yet another
+ * InputStream and prevent the outer layer from closing the inner InputStream
+ */
+public class NonCloseableInputStream extends FilterInputStream {
+
+ private final InputStream toWrap;
+
+ public NonCloseableInputStream(final InputStream toWrap) {
+ super(toWrap);
+ this.toWrap = toWrap;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return toWrap.read();
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return toWrap.read(b);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return toWrap.read(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // do nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java
new file mode 100644
index 0000000..9c77637
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class NonCloseableOutputStream extends FilterOutputStream {
+
+ private final OutputStream out;
+
+ public NonCloseableOutputStream(final OutputStream out) {
+ super(out);
+ this.out = out;
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java
new file mode 100644
index 0000000..8452761
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * OutputStream that throws away all data, just like as if writing to /dev/null
+ */
+public class NullOutputStream extends OutputStream {
+
+ @Override
+ public void write(final int b) throws IOException {
+ }
+
+ @Override
+ public void write(final byte[] b) throws IOException {
+ }
+
+ @Override
+ public void write(final byte[] b, int off, int len) throws IOException {
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void flush() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java
new file mode 100644
index 0000000..9158050
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public interface StreamThrottler extends Closeable {
+
+ long copy(InputStream in, OutputStream out) throws IOException;
+
+ long copy(InputStream in, OutputStream out, long maxBytes) throws IOException;
+
+ InputStream newThrottledInputStream(final InputStream toWrap);
+
+ OutputStream newThrottledOutputStream(final OutputStream toWrap);
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java
new file mode 100644
index 0000000..8e3d606
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.stream.io.exception.BytePatternNotFoundException;
+import org.apache.nifi.stream.io.util.NonThreadSafeCircularBuffer;
+
+public class StreamUtils {
+
+ public static long copy(final InputStream source, final OutputStream destination) throws IOException {
+ final byte[] buffer = new byte[8192];
+ int len;
+ long totalCount = 0L;
+ while ((len = source.read(buffer)) > 0) {
+ destination.write(buffer, 0, len);
+ totalCount += len;
+ }
+ return totalCount;
+ }
+
+ /**
+ * Copies <code>numBytes</code> from <code>source</code> to
+ * <code>destination</code>. If <code>numBytes</code> are not available from
+ * <code>source</code>, throws EOFException
+ *
+ * @param source
+ * @param destination
+ * @param numBytes
+ * @throws IOException
+ */
+ public static void copy(final InputStream source, final OutputStream destination, final long numBytes) throws IOException {
+ final byte[] buffer = new byte[8192];
+ int len;
+ long bytesLeft = numBytes;
+ while ((len = source.read(buffer, 0, (int) Math.min(bytesLeft, buffer.length))) > 0) {
+ destination.write(buffer, 0, len);
+ bytesLeft -= len;
+ }
+
+ if (bytesLeft > 0) {
+ throw new EOFException("Attempted to copy " + numBytes + " bytes but only " + (numBytes - bytesLeft) + " bytes were available");
+ }
+ }
+
+ /**
+ * Reads data from the given input stream, copying it to the destination
+ * byte array. If the InputStream has less data than the given byte array,
+ * throws an EOFException
+ *
+ * @param source
+ * @param destination
+ * @throws IOException
+ */
+ public static void fillBuffer(final InputStream source, final byte[] destination) throws IOException {
+ fillBuffer(source, destination, true);
+ }
+
+ /**
+ * Reads data from the given input stream, copying it to the destination
+ * byte array. If the InputStream has less data than the given byte array,
+ * throws an EOFException if <code>ensureCapacity</code> is true and
+ * otherwise returns the number of bytes copied
+ *
+ * @param source
+ * @param destination
+ * @param ensureCapacity whether or not to enforce that the InputStream have
+ * at least as much data as the capacity of the destination byte array
+ * @return
+ * @throws IOException
+ */
+ public static int fillBuffer(final InputStream source, final byte[] destination, final boolean ensureCapacity) throws IOException {
+ int bytesRead = 0;
+ int len;
+ while (bytesRead < destination.length) {
+ len = source.read(destination, bytesRead, destination.length - bytesRead);
+ if (len < 0) {
+ if (ensureCapacity) {
+ throw new EOFException();
+ } else {
+ break;
+ }
+ }
+
+ bytesRead += len;
+ }
+
+ return bytesRead;
+ }
+
+ /**
+ * Copies data from in to out until either we are out of data (returns null)
+ * or we hit one of the byte patterns identified by the
+ * <code>stoppers</code> parameter (returns the byte pattern matched). The
+ * bytes in the stopper will be copied.
+ *
+ * @param in
+ * @param out
+ * @param maxBytes
+ * @param stoppers
+ * @return the byte array matched, or null if end of stream was reached
+ * @throws IOException
+ */
+ public static byte[] copyInclusive(final InputStream in, final OutputStream out, final int maxBytes, final byte[]... stoppers) throws IOException {
+ if (stoppers.length == 0) {
+ return null;
+ }
+
+ final List<NonThreadSafeCircularBuffer> circularBuffers = new ArrayList<NonThreadSafeCircularBuffer>();
+ for (final byte[] stopper : stoppers) {
+ circularBuffers.add(new NonThreadSafeCircularBuffer(stopper));
+ }
+
+ long bytesRead = 0;
+ while (true) {
+ final int next = in.read();
+ if (next == -1) {
+ return null;
+ } else if (maxBytes > 0 && ++bytesRead >= maxBytes) {
+ throw new BytePatternNotFoundException("Did not encounter any byte pattern that was expected; data does not appear to be in the expected format");
+ }
+
+ out.write(next);
+
+ for (final NonThreadSafeCircularBuffer circ : circularBuffers) {
+ if (circ.addAndCompare((byte) next)) {
+ return circ.getByteArray();
+ }
+ }
+ }
+ }
+
+ /**
+ * Copies data from in to out until either we are out of data (returns null)
+ * or we hit one of the byte patterns identified by the
+ * <code>stoppers</code> parameter (returns the byte pattern matched). The
+ * byte pattern matched will NOT be copied to the output and will be un-read
+ * from the input.
+ *
+ * @param in
+ * @param out
+ * @param maxBytes
+ * @param stoppers
+ * @return the byte array matched, or null if end of stream was reached
+ * @throws IOException
+ */
+ public static byte[] copyExclusive(final InputStream in, final OutputStream out, final int maxBytes, final byte[]... stoppers) throws IOException {
+ if (stoppers.length == 0) {
+ return null;
+ }
+
+ int longest = 0;
+ NonThreadSafeCircularBuffer longestBuffer = null;
+ final List<NonThreadSafeCircularBuffer> circularBuffers = new ArrayList<NonThreadSafeCircularBuffer>();
+ for (final byte[] stopper : stoppers) {
+ final NonThreadSafeCircularBuffer circularBuffer = new NonThreadSafeCircularBuffer(stopper);
+ if (stopper.length > longest) {
+ longest = stopper.length;
+ longestBuffer = circularBuffer;
+ circularBuffers.add(0, circularBuffer);
+ } else {
+ circularBuffers.add(circularBuffer);
+ }
+ }
+
+ long bytesRead = 0;
+ while (true) {
+ final int next = in.read();
+ if (next == -1) {
+ return null;
+ } else if (maxBytes > 0 && bytesRead++ > maxBytes) {
+ throw new BytePatternNotFoundException("Did not encounter any byte pattern that was expected; data does not appear to be in the expected format");
+ }
+
+ for (final NonThreadSafeCircularBuffer circ : circularBuffers) {
+ if (circ.addAndCompare((byte) next)) {
+ // The longest buffer has some data that may not have been written out yet; we need to make sure
+ // that we copy out those bytes.
+ final int bytesToCopy = longest - circ.getByteArray().length;
+ for (int i = 0; i < bytesToCopy; i++) {
+ final int oldestByte = longestBuffer.getOldestByte();
+ if (oldestByte != -1) {
+ out.write(oldestByte);
+ longestBuffer.addAndCompare((byte) 0);
+ }
+ }
+
+ return circ.getByteArray();
+ }
+ }
+
+ if (longestBuffer.isFilled()) {
+ out.write(longestBuffer.getOldestByte());
+ }
+ }
+ }
+
+ /**
+ * Skips the specified number of bytes from the InputStream
+ *
+ * If unable to skip that number of bytes, throws EOFException
+ *
+ * @param stream
+ * @param bytesToSkip
+ * @throws IOException
+ */
+ public static void skip(final InputStream stream, final long bytesToSkip) throws IOException {
+ if (bytesToSkip <= 0) {
+ return;
+ }
+ long totalSkipped = 0L;
+
+ // If we have a FileInputStream, calling skip(1000000) will return 1000000 even if the file is only
+ // 3 bytes. As a result, we will skip 1 less than the number requested, and then read the last
+ // byte in order to make sure that we've consumed the number of bytes requested. We then check that
+ // the final byte, which we read, is not -1.
+ final long actualBytesToSkip = bytesToSkip - 1;
+ while (totalSkipped < actualBytesToSkip) {
+ final long skippedThisIteration = stream.skip(actualBytesToSkip - totalSkipped);
+ if (skippedThisIteration == 0) {
+ final int nextByte = stream.read();
+ if (nextByte == -1) {
+ throw new EOFException();
+ } else {
+ totalSkipped++;
+ }
+ }
+
+ totalSkipped += skippedThisIteration;
+ }
+
+ final int lastByte = stream.read();
+ if (lastByte == -1) {
+ throw new EOFException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java
new file mode 100644
index 0000000..2b9050d
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.OutputStream;
+
+/**
+ * This class extends the {@link java.util.zip.ZipOutputStream} by providing a
+ * constructor that allows the user to specify the compression level. The
+ * default compression level is 1, as opposed to Java's default of 5.
+ */
+public class ZipOutputStream extends java.util.zip.ZipOutputStream {
+
+ public static final int DEFAULT_COMPRESSION_LEVEL = 1;
+
+ public ZipOutputStream(final OutputStream out) {
+ this(out, DEFAULT_COMPRESSION_LEVEL);
+ }
+
+ public ZipOutputStream(final OutputStream out, final int compressionLevel) {
+ super(out);
+ def.setLevel(compressionLevel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java
new file mode 100644
index 0000000..5d08616
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io.exception;
+
+import java.io.IOException;
+
+public class BytePatternNotFoundException extends IOException {
+
+ private static final long serialVersionUID = -4128911284318513973L;
+
+ public BytePatternNotFoundException(final String explanation) {
+ super(explanation);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java
new file mode 100644
index 0000000..b4b4c17
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io.util;
+
+import java.util.Arrays;
+
+public class NonThreadSafeCircularBuffer {
+
+ private final byte[] lookingFor;
+ private final int[] buffer;
+ private int insertionPointer = 0;
+ private int bufferSize = 0;
+
+ public NonThreadSafeCircularBuffer(final byte[] lookingFor) {
+ this.lookingFor = lookingFor;
+ buffer = new int[lookingFor.length];
+ Arrays.fill(buffer, -1);
+ }
+
+ public byte[] getByteArray() {
+ return lookingFor;
+ }
+
+ /**
+ * Returns the oldest byte in the buffer
+ *
+ * @return
+ */
+ public int getOldestByte() {
+ return buffer[insertionPointer];
+ }
+
+ public boolean isFilled() {
+ return bufferSize >= buffer.length;
+ }
+
+ public boolean addAndCompare(final byte data) {
+ buffer[insertionPointer] = data;
+ insertionPointer = (insertionPointer + 1) % lookingFor.length;
+
+ bufferSize++;
+ if (bufferSize < lookingFor.length) {
+ return false;
+ }
+
+ for (int i = 0; i < lookingFor.length; i++) {
+ final byte compare = (byte) buffer[(insertionPointer + i) % lookingFor.length];
+ if (compare != lookingFor[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1357a17b/commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
----------------------------------------------------------------------
diff --git a/commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java b/commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
new file mode 100644
index 0000000..85bfd96
--- /dev/null
+++ b/commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Arrays;
+
+/**
+ * <p>
+ * A RingBuffer that can be used to scan byte sequences for subsequences.
+ * </p>
+ *
+ * <p>
+ * This class implements an efficient naive search algorithm, which allows the
+ * user of the library to identify byte sequences in a stream on-the-fly so that
+ * the stream can be segmented without having to buffer the data.
+ * </p>
+ *
+ * <p>
+ * The intended usage paradigm is:
+ * <code>
+ * <pre>
+ * final byte[] searchSequence = ...;
+ * final CircularBuffer buffer = new CircularBuffer(searchSequence);
+ * while ((int nextByte = in.read()) > 0) {
+ * if ( buffer.addAndCompare(nextByte) ) {
+ * // This byte is the last byte in the given sequence
+ * } else {
+ * // This byte does not complete the given sequence
+ * }
+ * }
+ * </pre>
+ * </code>
+ * </p>
+ */
+public class NaiveSearchRingBuffer {
+
+ private final byte[] lookingFor;
+ private final int[] buffer;
+ private int insertionPointer = 0;
+ private int bufferSize = 0;
+
+ public NaiveSearchRingBuffer(final byte[] lookingFor) {
+ this.lookingFor = lookingFor;
+ this.buffer = new int[lookingFor.length];
+ Arrays.fill(buffer, -1);
+ }
+
+ /**
+ * Returns the contents of the internal buffer, which represents the last X
+ * bytes added to the buffer, where X is the minimum of the number of bytes
+ * added to the buffer or the length of the byte sequence for which we are
+ * looking
+ *
+ * @return
+ */
+ public byte[] getBufferContents() {
+ final int contentLength = Math.min(lookingFor.length, bufferSize);
+ final byte[] contents = new byte[contentLength];
+ for (int i = 0; i < contentLength; i++) {
+ final byte nextByte = (byte) buffer[(insertionPointer + i) % lookingFor.length];
+ contents[i] = nextByte;
+ }
+ return contents;
+ }
+
+ /**
+ * Returns the oldest byte in the buffer
+ *
+ * @return
+ */
+ public int getOldestByte() {
+ return buffer[insertionPointer];
+ }
+
+ /**
+ * Returns <code>true</code> if the number of bytes that have been added to
+ * the buffer is at least equal to the length of the byte sequence for which
+ * we are searching
+ *
+ * @return
+ */
+ public boolean isFilled() {
+ return bufferSize >= buffer.length;
+ }
+
+ /**
+ * Clears the internal buffer so that a new search may begin
+ */
+ public void clear() {
+ Arrays.fill(buffer, -1);
+ insertionPointer = 0;
+ bufferSize = 0;
+ }
+
+ /**
+ * Add the given byte to the buffer and notify whether or not the byte
+ * completes the desired byte sequence.
+ *
+ * @param data
+ * @return <code>true</code> if this byte completes the byte sequence,
+ * <code>false</code> otherwise.
+ */
+ public boolean addAndCompare(final byte data) {
+ buffer[insertionPointer] = data;
+ insertionPointer = (insertionPointer + 1) % lookingFor.length;
+
+ bufferSize++;
+ if (bufferSize < lookingFor.length) {
+ return false;
+ }
+
+ for (int i = 0; i < lookingFor.length; i++) {
+ final byte compare = (byte) buffer[(insertionPointer + i) % lookingFor.length];
+ if (compare != lookingFor[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}