You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2016/10/21 19:58:46 UTC
[01/27] nifi git commit: NIFI-2873: Nifi throws UnknownHostException
with HA NameNode [Forced Update!]
Repository: nifi
Updated Branches:
refs/heads/appveyor-improvement cde49cc6f -> 81313dda6 (forced update)
NIFI-2873: Nifi throws UnknownHostException with HA NameNode
Signed-off-by: Matt Burgess <ma...@apache.org>
NIFI-2873: Changed test hive-site.xml to use local FS, fixed Checkstyle violations
This closes #1113
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/e969a5ff
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/e969a5ff
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/e969a5ff
Branch: refs/heads/appveyor-improvement
Commit: e969a5ffe3a1e03b0b6a0f0a11e954dc396bf25a
Parents: 7107616
Author: d810146 <na...@team.telstra.com>
Authored: Fri Oct 7 13:27:30 2016 +1100
Committer: Matt Burgess <ma...@apache.org>
Committed: Fri Oct 14 09:23:19 2016 -0400
----------------------------------------------------------------------
.../org/apache/nifi/processors/hive/PutHiveStreaming.java | 2 ++
.../java/org/apache/nifi/util/hive/HiveConfigurator.java | 9 +++++++++
.../nifi-hive-processors/src/test/resources/hive-site.xml | 2 +-
3 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e969a5ff/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java
index 84eda83..e53ddc9 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java
@@ -323,6 +323,8 @@ public class PutHiveStreaming extends AbstractProcessor {
.withMaxOpenConnections(maxConnections)
.withHeartBeatInterval(heartbeatInterval);
+ hiveConfigurator.preload(hiveConfig);
+
if (SecurityUtil.isSecurityEnabled(hiveConfig)) {
final String principal = context.getProperty(kerberosProperties.getKerberosPrincipal()).getValue();
final String keyTab = context.getProperty(kerberosProperties.getKerberosKeytab()).getValue();
http://git-wip-us.apache.org/repos/asf/nifi/blob/e969a5ff/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java
index 748847d..ad22177 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java
@@ -18,6 +18,7 @@ package org.apache.nifi.util.hive;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.UserGroupInformation;
@@ -71,6 +72,14 @@ public class HiveConfigurator {
return hiveConfig;
}
+ public void preload(Configuration configuration) {
+ try {
+ FileSystem.get(configuration);
+ } catch (IOException ioe) {
+ // Suppress exception as future uses of this configuration will fail
+ }
+ }
+
public UserGroupInformation authenticate(final Configuration hiveConfig, String principal, String keyTab, long ticketRenewalPeriod, ComponentLog log) throws AuthenticationFailedException {
UserGroupInformation ugi;
http://git-wip-us.apache.org/repos/asf/nifi/blob/e969a5ff/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site.xml
index 52e32a4..7e7f86c 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site.xml
@@ -17,6 +17,6 @@
<configuration>
<property>
<name>fs.default.name</name>
- <value>hdfs://hive</value>
+ <value>file:///</value>
</property>
</configuration>
\ No newline at end of file
[08/27] nifi git commit: [NIFI-2898] Add Processor dialog processor
descriptions now scrollable
Posted by al...@apache.org.
[NIFI-2898] Add Processor dialog processor descriptions now scrollable
This closes #1134.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9fb30394
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9fb30394
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9fb30394
Branch: refs/heads/appveyor-improvement
Commit: 9fb3039416d05ea1e76e530c29527d466927e1d1
Parents: b9a940b
Author: Scott Aslan <sc...@gmail.com>
Authored: Thu Oct 13 16:55:17 2016 -0400
Committer: Pierre Villard <pi...@gmail.com>
Committed: Mon Oct 17 23:07:39 2016 +0200
----------------------------------------------------------------------
.../nifi-web-ui/src/main/webapp/css/new-processor-dialog.css | 1 +
.../js/nf/canvas/header/components/nf-ng-processor-component.js | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9fb30394/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-processor-dialog.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-processor-dialog.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-processor-dialog.css
index b95d2af..4444b07 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-processor-dialog.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-processor-dialog.css
@@ -61,6 +61,7 @@
height: 62px;
font-size: 12px;
line-height: 16px;
+ overflow-y: auto;
}
#processor-types-table {
http://git-wip-us.apache.org/repos/asf/nifi/blob/9fb30394/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-processor-component.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-processor-component.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-processor-component.js
index 387bfd4..08e78cd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-processor-component.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-processor-component.js
@@ -310,6 +310,8 @@ nf.ng.ProcessorComponent = function (serviceProvider) {
$('#processor-type-description').html(processorType.description).ellipsis();
}
+ nf.Common.toggleScrollable($('#processor-type-description').get(0));
+
// populate the dom
$('#processor-type-name').text(processorType.label).ellipsis();
$('#selected-processor-name').text(processorType.label);
[13/27] nifi git commit: [NIFI-1459] add css translate3d to
properties table editors in order to move the element along the z-axis of the
3D space and allow the scrollbars to properly be poistioned. This closes
#1070
Posted by al...@apache.org.
[NIFI-1459] add css translate3d to properties table editors in order to move the element along the z-axis of the 3D space and allow the scrollbars to properly be poistioned. This closes #1070
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a8e1c775
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a8e1c775
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a8e1c775
Branch: refs/heads/appveyor-improvement
Commit: a8e1c775fd95cfb73ab5453a8e7ec7c1d3e03b98
Parents: 3914141
Author: Scott Aslan <sc...@gmail.com>
Authored: Wed Oct 19 11:57:29 2016 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Oct 19 15:39:58 2016 -0400
----------------------------------------------------------------------
.../js/jquery/propertytable/jquery.propertytable.js | 12 +++++++++---
.../src/main/webapp/js/application.js | 6 ++++--
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8e1c775/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
index 47231f0..246129c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/propertytable/jquery.propertytable.js
@@ -77,7 +77,8 @@
'background-color': 'rgb(255, 255, 255)',
'overflow': 'hidden',
'padding': '10px 20px',
- 'cursor': 'move'
+ 'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)'
}).appendTo(container);
// create the input field
@@ -301,7 +302,8 @@
'border-radius': '2px',
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
'background-color': 'rgb(255, 255, 255)',
- 'cursor': 'move'
+ 'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)'
}).draggable({
cancel: 'input, textarea, pre, .nf-checkbox, .button, .' + editorClass,
containment: 'parent'
@@ -506,7 +508,8 @@
'border-radius': '2px',
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
'background-color': 'rgb(255, 255, 255)',
- 'cursor': 'move'
+ 'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)'
}).draggable({
cancel: '.button, .combo',
containment: 'parent'
@@ -723,6 +726,7 @@
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
'background-color': 'rgb(255, 255, 255)',
'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)',
'top': offset.top - 24,
'left': offset.left - 20
}).appendTo('body');
@@ -805,6 +809,7 @@
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
'background-color': 'rgb(255, 255, 255)',
'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)',
'top': offset.top - 22,
'left': offset.left - 43
}).draggable({
@@ -835,6 +840,7 @@
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
'background-color': 'rgb(255, 255, 255)',
'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)',
'top': offset.top - 26,
'left': offset.left - 20
});
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8e1c775/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/js/application.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/js/application.js b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/js/application.js
index 1963f3d..a7984d7 100644
--- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/js/application.js
+++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/js/application.js
@@ -1727,7 +1727,8 @@ var ua = {
'overflow': 'hidden',
'border-radius': '2px',
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
- 'cursor': 'move'
+ 'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)'
}).draggable({
containment: 'parent'
}).appendTo(container);
@@ -1865,7 +1866,8 @@ var ua = {
'overflow': 'hidden',
'border-radius': '2px',
'box-shadow': 'rgba(0, 0, 0, 0.247059) 0px 2px 5px',
- 'cursor': 'move'
+ 'cursor': 'move',
+ 'transform': 'translate3d(0px, 0px, 0px)'
}).draggable({
cancel: 'input, textarea, pre, .button, div.' + editorClass,
containment: 'parent'
[03/27] nifi git commit: NIFI-2897: Fixed SelectHiveQL for CSV output
of complex types
Posted by al...@apache.org.
NIFI-2897: Fixed SelectHiveQL for CSV output of complex types
This closes #1132
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b52b8398
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b52b8398
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b52b8398
Branch: refs/heads/appveyor-improvement
Commit: b52b8398957dcb60b93ea5f697f45d3e0e212082
Parents: 88d1251
Author: Matt Burgess <ma...@apache.org>
Authored: Thu Oct 13 13:27:49 2016 -0400
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Fri Oct 14 12:35:38 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/nifi/util/hive/HiveJdbcCommon.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b52b8398/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java
index d4b2945..689baf9 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java
@@ -339,6 +339,16 @@ public class HiveJdbcCommon {
rowValues.add("");
}
break;
+ case ARRAY:
+ case STRUCT:
+ case JAVA_OBJECT:
+ String complexValueString = rs.getString(i);
+ if (complexValueString != null) {
+ rowValues.add(StringEscapeUtils.escapeCsv(complexValueString));
+ } else {
+ rowValues.add("");
+ }
+ break;
default:
if (value != null) {
rowValues.add(value.toString());
[10/27] nifi git commit: [NIFI-2913] update z-index of codemirror
resize handle to match the z-index of the codemirror vert. and horz.
scrollbars
Posted by al...@apache.org.
[NIFI-2913] update z-index of codemirror resize handle to match the z-index of the codemirror vert. and horz. scrollbars
This closes #1145.
Signed-off-by: Andy LoPresto <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f287bc91
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f287bc91
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f287bc91
Branch: refs/heads/appveyor-improvement
Commit: f287bc9119c2f0d018329f53c85fb502b5c96f18
Parents: 5afbc43
Author: Scott Aslan <sc...@gmail.com>
Authored: Tue Oct 18 14:43:48 2016 -0400
Committer: Andy LoPresto <al...@apache.org>
Committed: Tue Oct 18 15:24:18 2016 -0700
----------------------------------------------------------------------
.../src/main/webapp/js/jquery/nfeditor/jquery.nfeditor.css | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f287bc91/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/nfeditor/jquery.nfeditor.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/nfeditor/jquery.nfeditor.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/nfeditor/jquery.nfeditor.css
index 3c0a6d9..dcf1457 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/nfeditor/jquery.nfeditor.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/nfeditor/jquery.nfeditor.css
@@ -38,6 +38,7 @@
height: 16px;
background-image: url(../../../images/iconResize.png);
background-color: #fff;
+ z-index: 6;
}
/*
[23/27] nifi git commit: Adjusting test to use a temporary folder to
alleviate Windows concerns with provenance testing.
Posted by al...@apache.org.
Adjusting test to use a temporary folder to alleviate Windows concerns with provenance testing.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/df61cde3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/df61cde3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/df61cde3
Branch: refs/heads/appveyor-improvement
Commit: df61cde3784b40ba7d1dd0b74e049db547f8af9a
Parents: af10118
Author: Aldrin Piri <al...@apache.org>
Authored: Fri Oct 14 11:09:25 2016 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Fri Oct 21 15:54:33 2016 -0400
----------------------------------------------------------------------
.../TestPersistentProvenanceRepository.java | 32 +++++++++++---------
1 file changed, 17 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/df61cde3/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
index 80a67eb..7c8a6dc 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
@@ -85,6 +85,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestName;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
@@ -97,6 +98,9 @@ public class TestPersistentProvenanceRepository {
@Rule
public TestName name = new TestName();
+ @Rule
+ TemporaryFolder testFolder = new TemporaryFolder();
+
private PersistentProvenanceRepository repo;
private RepositoryConfiguration config;
@@ -104,9 +108,9 @@ public class TestPersistentProvenanceRepository {
private EventReporter eventReporter;
private List<ReportedEvent> reportedEvents = Collections.synchronizedList(new ArrayList<ReportedEvent>());
- private RepositoryConfiguration createConfiguration() {
+ private RepositoryConfiguration createConfiguration() throws IOException {
config = new RepositoryConfiguration();
- config.addStorageDirectory(new File("target/storage/" + UUID.randomUUID().toString()));
+ config.addStorageDirectory(testFolder.newFolder());
config.setCompressOnRollover(true);
config.setMaxEventFileLife(2000L, TimeUnit.SECONDS);
config.setCompressionBlockBytes(100);
@@ -169,7 +173,6 @@ public class TestPersistentProvenanceRepository {
}
-
private EventReporter getEventReporter() {
return eventReporter;
}
@@ -360,7 +363,7 @@ public class TestPersistentProvenanceRepository {
int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
- for (int i=0; i < immenseAttrSize; i++) {
+ for (int i = 0; i < immenseAttrSize; i++) {
immenseBldr.append('0');
}
final String uuid = "00000000-0000-0000-0000-000000000000";
@@ -1491,9 +1494,9 @@ public class TestPersistentProvenanceRepository {
assertEquals(5, lineageNodes.stream().map(node -> node.getNodeType()).filter(t -> t == LineageNodeType.PROVENANCE_EVENT_NODE).count());
final Set<EventNode> eventNodes = lineageNodes.stream()
- .filter(node -> node.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE)
- .map(node -> (EventNode) node)
- .collect(Collectors.toSet());
+ .filter(node -> node.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE)
+ .map(node -> (EventNode) node)
+ .collect(Collectors.toSet());
final Map<ProvenanceEventType, List<EventNode>> nodesByType = eventNodes.stream().collect(Collectors.groupingBy(EventNode::getEventType));
assertEquals(1, nodesByType.get(ProvenanceEventType.RECEIVE).size());
@@ -1517,9 +1520,9 @@ public class TestPersistentProvenanceRepository {
assertEquals(3, expandChildNodes.stream().map(node -> node.getNodeType()).filter(t -> t == LineageNodeType.PROVENANCE_EVENT_NODE).count());
final Set<EventNode> childEventNodes = expandChildNodes.stream()
- .filter(node -> node.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE)
- .map(node -> (EventNode) node)
- .collect(Collectors.toSet());
+ .filter(node -> node.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE)
+ .map(node -> (EventNode) node)
+ .collect(Collectors.toSet());
final Map<ProvenanceEventType, List<EventNode>> childNodesByType = childEventNodes.stream().collect(Collectors.groupingBy(EventNode::getEventType));
assertEquals(1, childNodesByType.get(ProvenanceEventType.FORK).size());
@@ -1529,7 +1532,6 @@ public class TestPersistentProvenanceRepository {
}
-
@Test
public void testBackPressure() throws IOException, InterruptedException {
final RepositoryConfiguration config = createConfiguration();
@@ -1720,7 +1722,7 @@ public class TestPersistentProvenanceRepository {
final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileLife(3, TimeUnit.SECONDS);
- repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS){
+ repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
@Override
File mergeJournals(List<File> journalFiles, File suggestedMergeFile, EventReporter eventReporter) throws IOException {
retryAmount.incrementAndGet();
@@ -1766,7 +1768,7 @@ public class TestPersistentProvenanceRepository {
exec.awaitTermination(10, TimeUnit.SECONDS);
repo.waitForRollover();
- assertEquals(5,retryAmount.get());
+ assertEquals(5, retryAmount.get());
}
@Test
@@ -1801,7 +1803,7 @@ public class TestPersistentProvenanceRepository {
}
- @Test(timeout=5000)
+ @Test(timeout = 5000)
public void testExceptionOnIndex() throws IOException {
final RepositoryConfiguration config = createConfiguration();
config.setMaxAttributeChars(50);
@@ -1840,7 +1842,7 @@ public class TestPersistentProvenanceRepository {
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
- for (int i=0; i < 1000; i++) {
+ for (int i = 0; i < 1000; i++) {
final ProvenanceEventRecord record = builder.build();
repo.registerEvent(record);
}
[27/27] nifi git commit: Removing MaxPermSize option.
Posted by al...@apache.org.
Removing MaxPermSize option.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/af101186
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/af101186
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/af101186
Branch: refs/heads/appveyor-improvement
Commit: af101186f55cabb458677c813032a0aa00fb3a23
Parents: 00d6fea
Author: Aldrin Piri <al...@apache.org>
Authored: Fri Oct 14 09:39:53 2016 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Fri Oct 21 15:54:33 2016 -0400
----------------------------------------------------------------------
appveyor.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/af101186/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index d54984f..b486557 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -29,8 +29,8 @@ install:
}
- cmd: SET M2_HOME=C:\maven\apache-maven-3.1.1
- cmd: SET PATH=%M2_HOME%\bin;%JAVA_HOME%\bin;%PATH%%
- - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g
- - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g
+ - cmd: SET MAVEN_OPTS=-Xmx4g
+ - cmd: SET JAVA_OPTS=-Xmx4g
build_script:
- mvn -q -T 2C -DskipTests clean install
test_script:
[07/27] nifi git commit: NIFI-2902 Fix for S3 Signer v4 Override
Posted by al...@apache.org.
NIFI-2902 Fix for S3 Signer v4 Override
This closes #1140.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b9a940bb
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b9a940bb
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b9a940bb
Branch: refs/heads/appveyor-improvement
Commit: b9a940bbd1c7afede946e2e7db948cc38582208f
Parents: c883f98
Author: James Wing <jv...@gmail.com>
Authored: Sat Oct 15 12:49:15 2016 -0700
Committer: Pierre Villard <pi...@gmail.com>
Committed: Mon Oct 17 22:51:45 2016 +0200
----------------------------------------------------------------------
.../processors/aws/s3/AbstractS3Processor.java | 2 +-
.../nifi/processors/aws/s3/TestPutS3Object.java | 64 ++++++++++++++++++++
2 files changed, 65 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b9a940bb/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
index 2f3fde8..b96c05e 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
@@ -121,7 +121,7 @@ public abstract class AbstractS3Processor extends AbstractAWSCredentialsProvider
.required(false)
.allowableValues(
new AllowableValue("Default Signature", "Default Signature"),
- new AllowableValue("AWSS3V4Signer", "Signature v4"),
+ new AllowableValue("AWSS3V4SignerType", "Signature v4"),
new AllowableValue("S3SignerType", "Signature v2"))
.defaultValue("Default Signature")
.build();
http://git-wip-us.apache.org/repos/asf/nifi/blob/b9a940bb/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
new file mode 100644
index 0000000..00f4423
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.util.List;
+
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.services.s3.AmazonS3Client;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+/**
+ * Unit tests for {@link PutS3Object}, without interaction with S3.
+ */
+public class TestPutS3Object {
+
+ @Test
+ public void testSignerOverrideOptions() {
+ final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
+ final ClientConfiguration config = new ClientConfiguration();
+ final PutS3Object processor = new PutS3Object();
+ final TestRunner runner = TestRunners.newTestRunner(processor);
+
+ final List<AllowableValue> allowableSignerValues = PutS3Object.SIGNER_OVERRIDE.getAllowableValues();
+ final String defaultSignerValue = PutS3Object.SIGNER_OVERRIDE.getDefaultValue();
+
+ for (AllowableValue allowableSignerValue : allowableSignerValues) {
+ String signerType = allowableSignerValue.getValue();
+ if (!signerType.equals(defaultSignerValue)) {
+ runner.setProperty(PutS3Object.SIGNER_OVERRIDE, signerType);
+ ProcessContext context = runner.getProcessContext();
+ try {
+ AmazonS3Client s3Client = processor.createClient(context, credentialsProvider, config);
+ } catch (IllegalArgumentException argEx) {
+ Assert.fail(argEx.getMessage());
+ }
+ }
+ }
+ }
+
+}
[18/27] nifi git commit: NIFI-2843: - Removing the View Configuration
menu item from the context menu on Process Groups.
Posted by al...@apache.org.
NIFI-2843: - Removing the View Configuration menu item from the context menu on Process Groups.
This closes #1087
Signed-off-by: jpercivall <jo...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/af27e407
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/af27e407
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/af27e407
Branch: refs/heads/appveyor-improvement
Commit: af27e4070b6ec2fb4bb7acfe4c2fd573849c802c
Parents: 99d3c39
Author: Matt Gilman <ma...@gmail.com>
Authored: Fri Sep 30 10:34:43 2016 -0400
Committer: jpercivall <jo...@yahoo.com>
Committed: Thu Oct 20 12:45:37 2016 -0400
----------------------------------------------------------------------
.../nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/af27e407/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js
index 56b6a71..1fb262e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js
@@ -967,9 +967,6 @@ nf.CanvasUtils = (function () {
return false;
}
- if (nf.CanvasUtils.isProcessGroup(selection)) {
- return true;
- }
if (nf.CanvasUtils.canRead(selection) === false) {
return false;
}
[05/27] nifi git commit: NIFI-2894: Fixed typo in PutSQL documentation
Posted by al...@apache.org.
NIFI-2894: Fixed typo in PutSQL documentation
This closes #1129
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/596b9886
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/596b9886
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/596b9886
Branch: refs/heads/appveyor-improvement
Commit: 596b98865b1c3526072ea120f86f0ff9627f82a8
Parents: d63e675
Author: Matt Burgess <ma...@apache.org>
Authored: Thu Oct 13 09:19:08 2016 -0400
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Fri Oct 14 12:45:34 2016 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/nifi/processors/standard/PutSQL.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/596b9886/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java
index eb27d40..adfac05 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java
@@ -116,7 +116,7 @@ public class PutSQL extends AbstractProcessor {
static final PropertyDescriptor SUPPORT_TRANSACTIONS = new PropertyDescriptor.Builder()
.name("Support Fragmented Transactions")
.description("If true, when a FlowFile is consumed by this Processor, the Processor will first check the fragment.identifier and fragment.count attributes of that FlowFile. "
- + "If the fragment.count value is greater than 1, the Processor will not process any FlowFile will that fragment.identifier until all are available; "
+ + "If the fragment.count value is greater than 1, the Processor will not process any FlowFile with that fragment.identifier until all are available; "
+ "at that point, it will process all FlowFiles with that fragment.identifier as a single transaction, in the order specified by the FlowFiles' fragment.index attributes. "
+ "This Provides atomicity of those SQL statements. If this value is false, these attributes will be ignored and the updates will occur independent of one another.")
.allowableValues("true", "false")
[02/27] nifi git commit: NIFI-2199 - allows nifi.sh restart through
ssh
Posted by al...@apache.org.
NIFI-2199 - allows nifi.sh restart through ssh
Closes: #1092
Signed-off-by: Andre F de Miranda <tr...@users.noreply.github.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/88d12513
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/88d12513
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/88d12513
Branch: refs/heads/appveyor-improvement
Commit: 88d125137e3bf3faf80b522edf5cf14f99419d60
Parents: e969a5f
Author: Michal Klempa <mi...@gmail.com>
Authored: Tue Oct 4 12:18:46 2016 +0200
Committer: Andre F de Miranda <tr...@users.noreply.github.com>
Committed: Sat Oct 15 00:48:04 2016 +1100
----------------------------------------------------------------------
.../nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/88d12513/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
index b58a622..79850ee 100755
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
@@ -267,7 +267,7 @@ run() {
RUN_NIFI_CMD="cd "\""${NIFI_HOME}"\"" && ${sudo_cmd_prefix} "\""${JAVA}"\"" -cp "\""${BOOTSTRAP_CLASSPATH}"\"" -Xms12m -Xmx24m ${BOOTSTRAP_DIR_PARAMS} org.apache.nifi.bootstrap.RunNiFi"
if [ "$1" = "start" ]; then
- (eval $RUN_NIFI_CMD $@ &)
+ (eval $RUN_NIFI_CMD $@ &)> /dev/null 2>&1 < /dev/null
else
(eval $RUN_NIFI_CMD $@)
fi
[17/27] nifi git commit: NIFI-1069 - improve init script exit codes
so that results are LSB compliant
Posted by al...@apache.org.
NIFI-1069 - improve init script exit codes so that results are LSB compliant
This closes: #1093
Signed-off-by: Andre F de Miranda <tr...@users.noreply.github.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/99d3c397
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/99d3c397
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/99d3c397
Branch: refs/heads/appveyor-improvement
Commit: 99d3c397481a98a5b3d2d4e44b84516fc0e3675e
Parents: 231f514
Author: Michal Klempa <mi...@gmail.com>
Authored: Tue Oct 4 14:25:49 2016 +0200
Committer: Andre F de Miranda <tr...@users.noreply.github.com>
Committed: Thu Oct 20 23:12:23 2016 +1100
----------------------------------------------------------------------
.../main/java/org/apache/nifi/bootstrap/RunNiFi.java | 15 ++++++++++-----
.../nifi-resources/src/main/resources/bin/nifi.sh | 2 ++
2 files changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/99d3c397/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
index 04e7ba3..8d92c44 100644
--- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
@@ -204,6 +204,7 @@ public class RunNiFi {
final File configFile = getBootstrapConfFile();
final RunNiFi runNiFi = new RunNiFi(configFile, verbose);
+ Integer exitStatus = null;
switch (cmd.toLowerCase()) {
case "start":
runNiFi.start();
@@ -215,7 +216,7 @@ public class RunNiFi {
runNiFi.stop();
break;
case "status":
- runNiFi.status();
+ exitStatus = runNiFi.status();
break;
case "restart":
runNiFi.stop();
@@ -228,6 +229,9 @@ public class RunNiFi {
runNiFi.env();
break;
}
+ if (exitStatus != null) {
+ System.exit(exitStatus);
+ }
}
private static File getBootstrapConfFile() {
@@ -581,23 +585,23 @@ public class RunNiFi {
return new Status(port, pid, pingSuccess, alive);
}
- public void status() throws IOException {
+ public int status() throws IOException {
final Logger logger = cmdLogger;
final Status status = getStatus(logger);
if (status.isRespondingToPing()) {
logger.info("Apache NiFi is currently running, listening to Bootstrap on port {}, PID={}",
new Object[]{status.getPort(), status.getPid() == null ? "unknown" : status.getPid()});
- return;
+ return 0;
}
if (status.isProcessRunning()) {
logger.info("Apache NiFi is running at PID {} but is not responding to ping requests", status.getPid());
- return;
+ return 4;
}
if (status.getPort() == null) {
logger.info("Apache NiFi is not running");
- return;
+ return 3;
}
if (status.getPid() == null) {
@@ -605,6 +609,7 @@ public class RunNiFi {
} else {
logger.info("Apache NiFi is not running");
}
+ return 3;
}
public void env() {
http://git-wip-us.apache.org/repos/asf/nifi/blob/99d3c397/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
index ff8addd..7b471d7 100755
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
@@ -295,12 +295,14 @@ run() {
else
(eval $RUN_NIFI_CMD $@)
fi
+ EXIT_STATUS=$?
# Wait just a bit (3 secs) to wait for the logging to finish and then echo a new-line.
# We do this to avoid having logs spewed on the console after running the command and then not giving
# control back to the user
sleep 3
echo
+ exit $EXIT_STATUS
}
main() {
[14/27] nifi git commit: NIFI-2919 Improved GetFile processor to fail
(and provide bulletins) if target directory is inaccessible.
Posted by al...@apache.org.
NIFI-2919 Improved GetFile processor to fail (and provide bulletins) if target directory is inaccessible.
This closes #1147.
Signed-off-by: Andy LoPresto <al...@apache.org>
Fixed typos in error messages, renamed variables in test, and cleaned up unnecessary imports. (+1 squashed commit)
Squashed commits:
[e755cbd] NIFI-2919 improved GetFile to fail if target directory is inaccessible
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/611cadd2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/611cadd2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/611cadd2
Branch: refs/heads/appveyor-improvement
Commit: 611cadd231309126a0a7737391f6e07730bb3864
Parents: a8e1c77
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Wed Oct 19 09:30:12 2016 -0400
Committer: Andy LoPresto <al...@apache.org>
Committed: Wed Oct 19 16:33:33 2016 -0400
----------------------------------------------------------------------
.../nifi/processors/standard/GetFile.java | 9 +-
.../nifi/processors/standard/TestGetFile.java | 101 +++++++++++++++----
2 files changed, 87 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/611cadd2/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFile.java
index 45c95eb..cdfb857 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFile.java
@@ -48,7 +48,6 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
-
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
@@ -295,14 +294,14 @@ public class GetFile extends AbstractProcessor {
}
private Set<File> performListing(final File directory, final FileFilter filter, final boolean recurseSubdirectories) {
+ Path p = directory.toPath();
+ if (!Files.isWritable(p) || !Files.isReadable(p)) {
+ throw new IllegalStateException("Directory '" + directory + "' does not have sufficient permissions (i.e., not writable and readable)");
+ }
final Set<File> queue = new HashSet<>();
if (!directory.exists()) {
return queue;
}
- // this check doesn't work on Windows
- if (!directory.canRead()) {
- getLogger().warn("No read permission on directory {}", new Object[]{directory.toString()});
- }
final File[] children = directory.listFiles();
if (children == null) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/611cadd2/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
index eb8a764..daf807a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
@@ -16,8 +16,6 @@
*/
package org.apache.nifi.processors.standard;
-import org.apache.nifi.processors.standard.GetFile;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -32,10 +30,10 @@ import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
-
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
@@ -45,6 +43,72 @@ import org.junit.Test;
public class TestGetFile {
@Test
+ public void testWithInaccessibleDir() throws IOException {
+ File inaccessibleDir = new File("target/inaccessible");
+ inaccessibleDir.deleteOnExit();
+ inaccessibleDir.mkdir();
+ Set<PosixFilePermission> posixFilePermissions = new HashSet<>();
+ Files.setPosixFilePermissions(inaccessibleDir.toPath(), posixFilePermissions);
+ final TestRunner runner = TestRunners.newTestRunner(new GetFile());
+ runner.setProperty(GetFile.DIRECTORY, inaccessibleDir.getAbsolutePath());
+ try {
+ runner.run();
+ fail();
+ } catch (AssertionError e) {
+ assertTrue(e.getCause().getMessage()
+ .endsWith("does not have sufficient permissions (i.e., not writable and readable)"));
+ }
+ }
+
+ @Test
+ public void testWithUnreadableDir() throws IOException {
+ File unreadableDir = new File("target/unreadable");
+ unreadableDir.deleteOnExit();
+ unreadableDir.mkdir();
+ Set<PosixFilePermission> posixFilePermissions = new HashSet<>();
+ posixFilePermissions.add(PosixFilePermission.GROUP_EXECUTE);
+ posixFilePermissions.add(PosixFilePermission.GROUP_WRITE);
+ posixFilePermissions.add(PosixFilePermission.OTHERS_EXECUTE);
+ posixFilePermissions.add(PosixFilePermission.OTHERS_WRITE);
+ posixFilePermissions.add(PosixFilePermission.OWNER_EXECUTE);
+ posixFilePermissions.add(PosixFilePermission.OWNER_WRITE);
+ Files.setPosixFilePermissions(unreadableDir.toPath(), posixFilePermissions);
+ final TestRunner runner = TestRunners.newTestRunner(new GetFile());
+ runner.setProperty(GetFile.DIRECTORY, unreadableDir.getAbsolutePath());
+ try {
+ runner.run();
+ fail();
+ } catch (AssertionError e) {
+ assertTrue(e.getCause().getMessage()
+ .endsWith("does not have sufficient permissions (i.e., not writable and readable)"));
+ }
+ }
+
+ @Test
+ public void testWithUnwritableDir() throws IOException {
+ File unwritableDir = new File("target/unwritable");
+ unwritableDir.deleteOnExit();
+ unwritableDir.mkdir();
+ Set<PosixFilePermission> posixFilePermissions = new HashSet<>();
+ posixFilePermissions.add(PosixFilePermission.GROUP_EXECUTE);
+ posixFilePermissions.add(PosixFilePermission.GROUP_READ);
+ posixFilePermissions.add(PosixFilePermission.OTHERS_EXECUTE);
+ posixFilePermissions.add(PosixFilePermission.OTHERS_READ);
+ posixFilePermissions.add(PosixFilePermission.OWNER_EXECUTE);
+ posixFilePermissions.add(PosixFilePermission.OWNER_READ);
+ Files.setPosixFilePermissions(unwritableDir.toPath(), posixFilePermissions);
+ final TestRunner runner = TestRunners.newTestRunner(new GetFile());
+ runner.setProperty(GetFile.DIRECTORY, unwritableDir.getAbsolutePath());
+ try {
+ runner.run();
+ fail();
+ } catch (AssertionError e) {
+ assertTrue(e.getCause().getMessage()
+ .endsWith("does not have sufficient permissions (i.e., not writable and readable)"));
+ }
+ }
+
+ @Test
public void testFilePickedUp() throws IOException {
final File directory = new File("target/test/data/in");
deleteDirectory(directory);
@@ -73,7 +137,7 @@ public class TestGetFile {
}
private void deleteDirectory(final File directory) throws IOException {
- if (directory.exists()) {
+ if (directory != null && directory.exists()) {
for (final File file : directory.listFiles()) {
if (file.isDirectory()) {
deleteDirectory(file);
@@ -155,29 +219,30 @@ public class TestGetFile {
try {
destFile.setLastModified(1000000000);
verifyLastModified = true;
- } catch (Exception donothing) {
+ } catch (Exception doNothing) {
}
boolean verifyPermissions = false;
try {
- // If you mount an NTFS partition in Linux, you are unable to change the permissions of the files,
- // because every file has the same permissions, controlled by the 'fmask' and 'dmask' mount options.
- // Executing a chmod command will not fail, but it does not change the file's permissions.
- // From Java perspective the NTFS mount point, as a FileStore supports the 'unix' and 'posix' file
- // attribute views, but the setPosixFilePermissions() has no effect.
- //
- // If you set verifyPermissions to true without the following extra check, the test case will fail
- // on a file system, where Nifi source is located on a NTFS mount point in Linux.
- // The purpose of the extra check is to ensure, that setPosixFilePermissions() changes the file's
- // permissions, and set verifyPermissions, after we are convinced.
+ /* If you mount an NTFS partition in Linux, you are unable to change the permissions of the files,
+ * because every file has the same permissions, controlled by the 'fmask' and 'dmask' mount options.
+ * Executing a chmod command will not fail, but it does not change the file's permissions.
+ * From Java perspective the NTFS mount point, as a FileStore supports the 'unix' and 'posix' file
+ * attribute views, but the setPosixFilePermissions() has no effect.
+ *
+ * If you set verifyPermissions to true without the following extra check, the test case will fail
+ * on a file system, where Nifi source is located on a NTFS mount point in Linux.
+ * The purpose of the extra check is to ensure, that setPosixFilePermissions() changes the file's
+ * permissions, and set verifyPermissions, after we are convinced.
+ */
Set<PosixFilePermission> perms = PosixFilePermissions.fromString("r--r-----");
Files.setPosixFilePermissions(targetPath, perms);
- Set<PosixFilePermission> permsAfterSet = Files.getPosixFilePermissions(targetPath);
+ Set<PosixFilePermission> permsAfterSet = Files.getPosixFilePermissions(targetPath);
if (perms.equals(permsAfterSet)) {
- verifyPermissions = true;
+ verifyPermissions = true;
}
- } catch (Exception donothing) {
+ } catch (Exception doNothing) {
}
final TestRunner runner = TestRunners.newTestRunner(new GetFile());
[06/27] nifi git commit: NIFI-2906 removed unused memebr variables
Posted by al...@apache.org.
NIFI-2906 removed unused memebr variables
This closes #1144.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c883f98c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c883f98c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c883f98c
Branch: refs/heads/appveyor-improvement
Commit: c883f98cbf803e5d8982a2a811c3ca46c5c249ca
Parents: 596b988
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Mon Oct 17 11:54:58 2016 -0400
Committer: Pierre Villard <pi...@gmail.com>
Committed: Mon Oct 17 22:32:23 2016 +0200
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/standard/ReplaceText.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/c883f98c/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
index 9b20ec6..59721f0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
@@ -210,8 +210,6 @@ public class ReplaceText extends AbstractProcessor {
final ComponentLog logger = getLogger();
- final String unsubstitutedRegex = context.getProperty(SEARCH_VALUE).getValue();
- String unsubstitutedReplacement = context.getProperty(REPLACEMENT_VALUE).getValue();
final String replacementStrategy = context.getProperty(REPLACEMENT_STRATEGY).getValue();
final Charset charset = Charset.forName(context.getProperty(CHARACTER_SET).getValue());
[21/27] nifi git commit: NIFI-2341 - Introduce ParseCEF processor
Posted by al...@apache.org.
NIFI-2341 - Introduce ParseCEF processor
This closes #785
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b864d49f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b864d49f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b864d49f
Branch: refs/heads/appveyor-improvement
Commit: b864d49f66692dabeb805965a08e83e14d27fd58
Parents: 9f6fb61
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Thu Aug 4 23:30:37 2016 +1000
Committer: Matt Burgess <ma...@apache.org>
Committed: Thu Oct 20 17:16:57 2016 -0400
----------------------------------------------------------------------
nifi-assembly/NOTICE | 5 +
.../src/main/resources/META-INF/NOTICE | 6 +
.../nifi-standard-processors/pom.xml | 11 +
.../nifi/processors/standard/ParseCEF.java | 327 +++++++++++++++++++
.../org.apache.nifi.processor.Processor | 1 +
.../nifi/processors/standard/TestParseCEF.java | 244 ++++++++++++++
6 files changed, 594 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b864d49f/nifi-assembly/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE
index 794a9d5..9d54885 100644
--- a/nifi-assembly/NOTICE
+++ b/nifi-assembly/NOTICE
@@ -1013,6 +1013,11 @@ The following binary components are provided under the Apache Software License v
Expert Group and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/
+ (ASLv2) ParCEFone
+ The following NOTICE information applies:
+ ParCEFone
+ Copyright 2016 Fluenda
+
This includes derived works from the Apache Software License V2 library python-evtx (https://github.com/williballenthin/python-evtx)
Copyright 2012, 2013 Willi Ballenthin william.ballenthin@mandiant.com
while at Mandiant http://www.mandiant.com
http://git-wip-us.apache.org/repos/asf/nifi/blob/b864d49f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
index 205d9ec..0887920 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
@@ -164,6 +164,11 @@ The following binary components are provided under the Apache Software License v
"GCC RUntime Library Exception"
http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
+ (ASLv2) ParCEFone
+ The following NOTICE information applies:
+ ParCEFone
+ Copyright 2016 Fluenda
+
************************
Common Development and Distribution License 1.1
************************
@@ -175,6 +180,7 @@ The following binary components are provided under the Common Development and Di
(CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.19 - https://jersey.java.net/jersey-server/)
(CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail)
(CDDL 1.1) (GPL2 w/ CPE) Javax JMS Api (javax.jms:javax.jms-api:jar:2.0.1 - http://java.net/projects/jms-spec/pages/Home)
+ (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 API (javax.el:javax.el-api:jar:3.0.0 - http://uel-spec.java.net)
*****************
Common Development and Distribution License v1.0:
http://git-wip-us.apache.org/repos/asf/nifi/blob/b864d49f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index b3bea5f..cce9c57 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -254,6 +254,17 @@ language governing permissions and limitations under the License. -->
<artifactId>org.everit.json.schema</artifactId>
<version>1.4.0</version>
</dependency>
+ <dependency>
+ <groupId>com.fluenda</groupId>
+ <artifactId>ParCEFone</artifactId>
+ <version>1.2.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b864d49f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
new file mode 100644
index 0000000..3417c0d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fluenda.parcefone.event.CEFHandlingException;
+import com.fluenda.parcefone.event.CommonEvent;
+import com.fluenda.parcefone.parser.CEFParser;
+
+import com.martiansoftware.macnificent.MacAddress;
+
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.text.SimpleDateFormat;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@InputRequirement(Requirement.INPUT_REQUIRED)
+@Tags({"logs", "cef", "attributes", "system", "event", "message"})
+@CapabilityDescription("Parses the contents of a CEF formatted message and adds attributes to the FlowFile for " +
+ "headers and extensions of the parts of the CEF message.\n" +
+ "Note: This Processor expects CEF messages WITHOUT the syslog headers (i.e. starting at \"CEF:0\"")
+@WritesAttributes({@WritesAttribute(attribute = "cef.header.version", description = "The version of the CEF message."),
+ @WritesAttribute(attribute = "cef.header.deviceVendor", description = "The Device Vendor of the CEF message."),
+ @WritesAttribute(attribute = "cef.header.deviceProduct", description = "The Device Product of the CEF message."),
+ @WritesAttribute(attribute = "cef.header.deviceVersion", description = "The Device Version of the CEF message."),
+ @WritesAttribute(attribute = "cef.header.deviceEventClassId", description = "The Device Event Class ID of the CEF message."),
+ @WritesAttribute(attribute = "cef.header.name", description = "The name of the CEF message."),
+ @WritesAttribute(attribute = "cef.header.severity", description = "The severity of the CEF message."),
+ @WritesAttribute(attribute = "cef.extension.*", description = "The key and value generated by the parsing of the message.")})
+@SeeAlso({ParseSyslog.class})
+
+public class ParseCEF extends AbstractProcessor {
+
+ // There should be no date format other than internationally agreed formats...
+ // flowfile-attributes uses Java 8 time to parse data (as Date objects are not timezoned)
+ private final static DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+
+ // for some reason Jackson doesnt seem to be able to use DateTieFormater
+ // so we use a SimpleDateFormat to format within flowfile-content
+ private final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+
+
+ // add a TZ object to be used by flowfile-attribute routine
+ private String tzId = null;
+
+ // Add serializer and mapper
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ public static final String DESTINATION_CONTENT = "flowfile-content";
+ public static final String DESTINATION_ATTRIBUTES = "flowfile-attribute";
+ public static final PropertyDescriptor FIELDS_DESTINATION = new PropertyDescriptor.Builder()
+ .name("FIELDS_DESTINATION")
+ .displayName("Parsed fields destination")
+ .description(
+ "Indicates whether the results of the CEF parser are written " +
+ "to the FlowFile content or a FlowFile attribute; if using " + DESTINATION_ATTRIBUTES +
+ "attribute, fields will be populated as attributes. " +
+ "If set to " + DESTINATION_CONTENT + ", the CEF extension field will be converted into " +
+ "a flat JSON object.")
+ .required(true)
+ .allowableValues(DESTINATION_CONTENT, DESTINATION_ATTRIBUTES)
+ .defaultValue(DESTINATION_CONTENT)
+ .build();
+
+ public static final PropertyDescriptor APPEND_RAW_MESSAGE_TO_JSON = new PropertyDescriptor.Builder()
+ .name("APPEND_RAW_MESSAGE_TO_JSON")
+ .displayName("Append raw message to JSON")
+ .description("When using flowfile-content (i.e. JSON output), add the original CEF message to " +
+ "the resulting JSON object. The original message is added as a string to _raw.")
+ .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+ .required(true)
+ .defaultValue("true")
+ .build();
+
+ public static final String UTC = "UTC";
+ public static final String LOCAL_TZ = "Local Timezone (system Default)";
+ public static final PropertyDescriptor TIME_REPRESENTATION = new PropertyDescriptor.Builder()
+ .name("TIME_REPRESENTATION")
+ .displayName("Timezone")
+ .description("Timezone to be used when representing date fields. UTC will convert all " +
+ "dates to UTC, while Local Timezone will convert them to the timezone used by NiFi.")
+ .allowableValues(UTC, LOCAL_TZ)
+ .required(true)
+ .defaultValue(LOCAL_TZ)
+ .build();
+
+ static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Any FlowFile that could not be parsed as a CEF message will be transferred to this Relationship without any attributes being added")
+ .build();
+ static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Any FlowFile that is successfully parsed as a CEF message will be to this Relationship.")
+ .build();
+
+ @Override
+ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor>properties =new ArrayList<>();
+ properties.add(FIELDS_DESTINATION);
+ properties.add(APPEND_RAW_MESSAGE_TO_JSON);
+ properties.add(TIME_REPRESENTATION);
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ final Set<Relationship> relationships = new HashSet<>();
+ relationships.add(REL_FAILURE);
+ relationships.add(REL_SUCCESS);
+ return relationships;
+ }
+
+ @OnScheduled
+ public void OnScheduled(final ProcessContext context) {
+
+ // Configure jackson mapper before spawning onTriggers
+ final SimpleModule module = new SimpleModule()
+ .addSerializer(MacAddress.class, new MacAddressToStringSerializer());
+ this.mapper.registerModule(module);
+ this.mapper.setDateFormat(this.simpleDateFormat);
+
+ switch (context.getProperty(TIME_REPRESENTATION).getValue()) {
+ case LOCAL_TZ:
+ // set the mapper TZ to local TZ
+ this.mapper.setTimeZone(TimeZone.getDefault());
+ tzId = TimeZone.getDefault().getID();
+ break;
+ case UTC:
+ // set the mapper TZ to local TZ
+ this.mapper.setTimeZone(TimeZone.getTimeZone(UTC));
+ tzId = UTC;
+ break;
+ }
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ final CEFParser parser = new CEFParser();
+ final byte[] buffer = new byte[(int) flowFile.getSize()];
+ session.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(final InputStream in) throws IOException {
+ StreamUtils.fillBuffer(in, buffer);
+ }
+ });
+
+ CommonEvent event;
+
+ try {
+ event = parser.parse(buffer, true);
+ } catch (Exception e) {
+ // This should never trigger but adding in here as a fencing mechanism to
+ // address possible ParCEFone bugs.
+ getLogger().error("Parser returned unexpected Exception {} while processing {}; routing to failure", new Object[] {e, flowFile});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+
+ // ParCEFone returns null every time it cannot parse an
+ // event, so we test
+ if (event==null) {
+ getLogger().error("Failed to parse {} as a CEF message: it does not conform to the CEF standard; routing to failure", new Object[] {flowFile});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+
+ try {
+ final String destination = context.getProperty(FIELDS_DESTINATION).getValue();
+
+ switch (destination) {
+ case DESTINATION_ATTRIBUTES:
+
+ final Map<String, String> attributes = new HashMap<>();
+
+ // Process KVs of the Header field
+ for (Map.Entry<String, Object> entry : event.getHeader().entrySet()) {
+ attributes.put("cef.header."+entry.getKey(), prettyResult(entry.getValue(), tzId));
+ }
+
+ // Process KVs composing the Extension field
+ for (Map.Entry<String, Object> entry : event.getExtension(true).entrySet()) {
+ attributes.put("cef.extension." + entry.getKey(), prettyResult(entry.getValue(), tzId));
+
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ }
+ break;
+
+ case DESTINATION_CONTENT:
+
+ ObjectNode results = mapper.createObjectNode();
+
+ // Add two JSON objects containing one CEF field each
+ results.set("header", mapper.valueToTree(event.getHeader()));
+ results.set("extension", mapper.valueToTree(event.getExtension(true)));
+
+ // Add the original content to original CEF content
+ // to the resulting JSON
+ if (context.getProperty(APPEND_RAW_MESSAGE_TO_JSON).asBoolean()) {
+ results.set("_raw", mapper.valueToTree(new String(buffer)));
+ }
+
+ flowFile = session.write(flowFile, new OutputStreamCallback() {
+ @Override
+ public void process(OutputStream out) throws IOException {
+ try (OutputStream outputStream = new BufferedOutputStream(out)) {
+ outputStream.write(mapper.writeValueAsBytes(results));
+ }
+ }
+ });
+
+ // Adjust the FlowFile mime.type attribute
+ flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json");
+
+ // Update the provenance for good measure
+ session.getProvenanceReporter().modifyContent(flowFile, "Replaced content with parsed CEF fields and values");
+ break;
+ }
+
+ // whatever the parsing stratgy, ready to transfer to success and commit
+ session.transfer(flowFile, REL_SUCCESS);
+ session.commit();
+ } catch (CEFHandlingException e) {
+ // The flowfile has failed parsing & validation, routing to failure and committing
+ getLogger().error("Failed to parse {} as a CEF message due to {}; routing to failure", new Object[] {flowFile, e});
+ session.transfer(flowFile, REL_FAILURE);
+ session.commit();
+ return;
+ } finally {
+ session.rollback();
+ }
+ }
+
+ private String prettyResult(Object entryValue, String tzID) {
+
+ if (entryValue instanceof InetAddress ) {
+ return ((InetAddress) entryValue).getHostAddress();
+ } else if (entryValue instanceof Date) {
+ ZonedDateTime zdt = ZonedDateTime.from(((Date) entryValue).toInstant().atZone(ZoneId.of(tzID)));
+ return(String.valueOf(zdt.format(dateTimeFormatter)));
+ } else {
+ return String.valueOf(entryValue);
+ }
+ }
+
+
+ // Serialize MacAddress as plain string
+ private class MacAddressToStringSerializer extends JsonSerializer<MacAddress> {
+
+ @Override
+ public void serialize(MacAddress macAddress,
+ JsonGenerator jsonGenerator,
+ SerializerProvider serializerProvider)
+ throws IOException, JsonProcessingException {
+ jsonGenerator.writeObject(macAddress.toString());
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b864d49f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index b27efdd..35d03d1 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -58,6 +58,7 @@ org.apache.nifi.processors.standard.LogAttribute
org.apache.nifi.processors.standard.MergeContent
org.apache.nifi.processors.standard.ModifyBytes
org.apache.nifi.processors.standard.MonitorActivity
+org.apache.nifi.processors.standard.ParseCEF
org.apache.nifi.processors.standard.ParseSyslog
org.apache.nifi.processors.standard.PostHTTP
org.apache.nifi.processors.standard.PutEmail
http://git-wip-us.apache.org/repos/asf/nifi/blob/b864d49f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
new file mode 100644
index 0000000..0da4d2b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+
+public class TestParseCEF {
+ private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+
+ private final static String sample1 = "CEF:0|TestVendor|TestProduct|TestVersion|TestEventClassID|TestName|Low|" +
+ // TimeStamp, String and Long
+ "rt=Feb 09 2015 00:27:43 UTC cn3Label=Test Long cn3=9223372036854775807 " +
+ // FloatPoint and MacAddress
+ "cfp1=1.234 cfp1Label=Test FP Number smac=00:00:0c:07:ac:00 " +
+ // IPv6 and String
+ "c6a3=2001:cdba::3257:9652 c6a3Label=Test IPv6 cs1Label=Test String cs1=test test test chocolate " +
+ // IPv4
+ "destinationTranslatedAddress=123.123.123.123 " +
+ // Date without TZ
+ "deviceCustomDate1=Feb 06 2015 13:27:43 " +
+ // Integer and IP Address (from v4)
+ "dpt=1234 agt=123.123.0.124 dlat=40.366633";
+
+ private final static String sample2 = "CEF:0|TestVendor|TestProduct|TestVersion|TestEventClassID|TestName|Low|" +
+ // TimeStamp, String and Long
+ "rt=Feb 09 2015 00:27:43 UTC cn3Label=Test Long cn3=9223372036854775807 " +
+ // FloatPoint and MacAddress
+ "cfp1=1.234 cfp1Label=Test FP Number smac=00:00:0c:07:ac:00 " +
+ // IPv6 and String
+ "c6a3=2001:cdba::3257:9652 c6a3Label=Test IPv6 cs1Label=Test String cs1=test test test chocolate " +
+ // IPv4
+ "destinationTranslatedAddress=123.123.123.123 " +
+ // Date without TZ
+ "deviceCustomDate1=Feb 06 2015 13:27:43 " +
+ // Integer and IP Address (from v4)
+ "dpt=1234 agt=123.123.0.124 dlat=40.366633 " +
+ // A JSON object inside one of CEF's custom Strings
+ "cs2Label=JSON payload " +
+ "cs2={\"test_test_test\": \"chocolate!\", \"what?!?\": \"Simple! test test test chocolate!\"}";
+
+
+ @Test
+ public void testInvalidMessage() {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.enqueue("test test test chocolate\n".getBytes());
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testSuccessfulParseToAttributes() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES);
+ runner.enqueue(sample1.getBytes());
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+ mff.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L)));
+ mff.assertAttributeEquals("cef.extension.cn3Label", "Test Long");
+ mff.assertAttributeEquals("cef.extension.cn3", "9223372036854775807");
+ mff.assertAttributeEquals("cef.extension.cfp1", "1.234");
+ mff.assertAttributeEquals("cef.extension.cfp1Label", "Test FP Number");
+ mff.assertAttributeEquals("cef.extension.smac", "00:00:0c:07:ac:00");
+ mff.assertAttributeEquals("cef.extension.c6a3", "2001:cdba:0:0:0:0:3257:9652");
+ mff.assertAttributeEquals("cef.extension.c6a3Label", "Test IPv6");
+ mff.assertAttributeEquals("cef.extension.cs1Label", "Test String");
+ mff.assertAttributeEquals("cef.extension.cs1", "test test test chocolate");
+ mff.assertAttributeEquals("cef.extension.destinationTranslatedAddress", "123.123.123.123");
+ mff.assertContentEquals(sample1.getBytes());
+
+
+ // Converting a field without timezone will always result on render time being dependent
+ // on locale of the machine running this test.
+ long eventTime = 1423229263000L;
+ int offset = TimeZone.getDefault().getOffset(eventTime);
+ sdf.setTimeZone(TimeZone.getDefault());
+
+ String prettyEvent = sdf.format(new Date(eventTime - offset));
+
+ mff.assertAttributeEquals("cef.extension.deviceCustomDate1",prettyEvent);
+ mff.assertAttributeEquals("cef.extension.dpt", "1234");
+ mff.assertAttributeEquals("cef.extension.agt", "123.123.0.124");
+ mff.assertAttributeEquals("cef.extension.dlat", "40.366633");
+ }
+
+ @Test
+ public void testSuccessfulParseToAttributesWithUTC() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES);
+ runner.setProperty(ParseCEF.TIME_REPRESENTATION, ParseCEF.UTC);
+ runner.enqueue(sample1.getBytes());
+ runner.run();
+
+ sdf.setTimeZone(TimeZone.getTimeZone(ParseCEF.UTC));
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+ mff.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L)));
+
+ // Converting a field without timezone will always result on render time being dependent
+ // on locale of the machine running this test.
+ long eventTime = 1423229263000L;
+ int offset = TimeZone.getDefault().getOffset(eventTime);
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ String prettyEvent = sdf.format(new Date(eventTime - offset));
+
+ mff.assertAttributeEquals("cef.extension.deviceCustomDate1",prettyEvent);
+ mff.assertContentEquals(sample1.getBytes());
+ }
+
+ @Test
+ public void testSuccessfulParseToContent() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT);
+ runner.enqueue(sample1.getBytes());
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+ byte [] rawJson = mff.toByteArray();
+
+ JsonNode results = new ObjectMapper().readTree(rawJson);
+
+ JsonNode header = results.get("header");
+ JsonNode extension = results.get("extension");
+
+ Assert.assertEquals("TestVendor", header.get("deviceVendor").asText());
+ Assert.assertEquals(sdf.format(new Date(1423441663000L)),
+ extension.get("rt").asText());
+ Assert.assertEquals("Test Long", extension.get("cn3Label").asText());
+ Assert.assertEquals( 9223372036854775807L, extension.get("cn3").asLong());
+ Assert.assertTrue(extension.get("cfp1").floatValue() == 1.234F);
+ Assert.assertEquals("Test FP Number", extension.get("cfp1Label").asText());
+ Assert.assertEquals("00:00:0c:07:ac:00", extension.get("smac").asText());
+ Assert.assertEquals("2001:cdba:0:0:0:0:3257:9652", extension.get("c6a3").asText());
+ Assert.assertEquals("Test IPv6", extension.get("c6a3Label").asText());
+ Assert.assertEquals("123.123.123.123", extension.get("destinationTranslatedAddress").asText());
+ Assert.assertEquals("Test String", extension.get("cs1Label").asText());
+ Assert.assertEquals("test test test chocolate", extension.get("cs1").asText());
+ }
+
+ @Test
+ public void testSuccessfulParseToContentWhenCEFContainsJSON() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT);
+ runner.enqueue(sample2.getBytes());
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+ byte [] rawJson = mff.toByteArray();
+
+ JsonNode results = new ObjectMapper().readTree(rawJson);
+
+ JsonNode header = results.get("header");
+ JsonNode extension = results.get("extension");
+
+ Assert.assertEquals("TestVendor", header.get("deviceVendor").asText());
+ Assert.assertEquals(sdf.format(new Date(1423441663000L)),
+ extension.get("rt").asText());
+ Assert.assertEquals("Test Long", extension.get("cn3Label").asText());
+ Assert.assertEquals( 9223372036854775807L, extension.get("cn3").asLong());
+ Assert.assertTrue(extension.get("cfp1").floatValue() == 1.234F);
+ Assert.assertEquals("Test FP Number", extension.get("cfp1Label").asText());
+ Assert.assertEquals("00:00:0c:07:ac:00", extension.get("smac").asText());
+ Assert.assertEquals("2001:cdba:0:0:0:0:3257:9652", extension.get("c6a3").asText());
+ Assert.assertEquals("Test IPv6", extension.get("c6a3Label").asText());
+ Assert.assertEquals("Test String", extension.get("cs1Label").asText());
+ Assert.assertEquals("test test test chocolate", extension.get("cs1").asText());
+ Assert.assertEquals("123.123.123.123", extension.get("destinationTranslatedAddress").asText());
+
+ JsonNode inner = new ObjectMapper().readTree(extension.get("cs2").asText());
+ Assert.assertEquals("chocolate!", inner.get("test_test_test").asText());
+ }
+
+
+ @Test
+ public void testSuccessfulParseToContentUTC() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT);
+ runner.setProperty(ParseCEF.TIME_REPRESENTATION, ParseCEF.UTC);
+ runner.enqueue(sample1.getBytes());
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+ byte [] rawJson = mff.toByteArray();
+
+ JsonNode results = new ObjectMapper().readTree(rawJson);
+
+ JsonNode extension = results.get("extension");
+
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Assert.assertEquals(sdf.format(new Date(1423441663000L)),
+ extension.get("rt").asText());
+
+ // Converting a field without timezone will always result on render time being dependent
+ // on locale of the machine running this test.
+ long eventTime = 1423229263000L;
+ int offset = TimeZone.getDefault().getOffset(eventTime);
+
+ // Set TZ to UTC
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ String prettyEvent = sdf.format(new Date(eventTime - offset));
+ Assert.assertEquals(prettyEvent, extension.get("deviceCustomDate1").asText());
+ }
+
+
+}
+
[19/27] nifi git commit: NIFI-2828: Fixed issue with transitive
Hadoop dependencies in Hive NAR
Posted by al...@apache.org.
NIFI-2828: Fixed issue with transitive Hadoop dependencies in Hive NAR
This closes #1146
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f5f132b1
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f5f132b1
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f5f132b1
Branch: refs/heads/appveyor-improvement
Commit: f5f132b1abc63a0b3612e4efe6a3faa129c337e2
Parents: af27e40
Author: Matt Burgess <ma...@apache.org>
Authored: Tue Oct 18 16:22:15 2016 -0400
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Thu Oct 20 14:56:01 2016 -0400
----------------------------------------------------------------------
.../nifi-hive-bundle/nifi-hive-processors/pom.xml | 7 +++++++
nifi-nar-bundles/nifi-hive-bundle/pom.xml | 5 +++++
pom.xml | 1 +
3 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f5f132b1/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
index 3d07b86..eebdcbf 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/pom.xml
@@ -58,6 +58,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
+ <version>${hive.hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
@@ -68,6 +69,12 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hadoop-utils</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.github.stephenc.findbugs</groupId>
http://git-wip-us.apache.org/repos/asf/nifi/blob/f5f132b1/nifi-nar-bundles/nifi-hive-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/pom.xml
index 5ab10e8..0186740 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/pom.xml
@@ -26,6 +26,11 @@
<version>1.1.0-SNAPSHOT</version>
<packaging>pom</packaging>
+ <properties>
+ <!-- Need to override hadoop.version here, for Hive and hadoop-client transitive dependencies -->
+ <hadoop.version>${hive.hadoop.version}</hadoop.version>
+ </properties>
+
<modules>
<module>nifi-hive-processors</module>
<module>nifi-hive-nar</module>
http://git-wip-us.apache.org/repos/asf/nifi/blob/f5f132b1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 735a9ec..7cf41b4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,6 +101,7 @@ language governing permissions and limitations under the License. -->
<yammer.metrics.version>2.2.0</yammer.metrics.version>
<ranger.version>0.6.0</ranger.version>
<hive.version>1.2.1</hive.version>
+ <hive.hadoop.version>2.6.2</hive.hadoop.version>
<hbase.version>1.1.2</hbase.version>
<storm.version>1.0.1</storm.version>
</properties>
[09/27] nifi git commit: NIFI-2500 This closes #1131. made container
queue configurable
Posted by al...@apache.org.
NIFI-2500 This closes #1131. made container queue configurable
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/5afbc433
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/5afbc433
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/5afbc433
Branch: refs/heads/appveyor-improvement
Commit: 5afbc433642a51a40c22a2048cfa7d0e9cf0a86a
Parents: 9fb3039
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Thu Oct 13 12:08:29 2016 -0400
Committer: joewitt <jo...@apache.org>
Committed: Tue Oct 18 14:47:49 2016 -0400
----------------------------------------------------------------------
.../processors/standard/HandleHttpRequest.java | 26 ++++++++++++++------
1 file changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/5afbc433/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index 9f8edfc..c1acc85 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -219,19 +219,20 @@ public class HandleHttpRequest extends AbstractProcessor {
.allowableValues(CLIENT_NONE, CLIENT_WANT, CLIENT_NEED)
.defaultValue(CLIENT_NONE.getValue())
.build();
+ public static final PropertyDescriptor CONTAINER_QUEUE_SIZE = new PropertyDescriptor.Builder()
+ .name("container-queue-size").displayName("Container Queue Size")
+ .description("The size of the queue for Http Request Containers").required(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR).defaultValue("50").build();
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("All content that is received is routed to the 'success' relationship")
.build();
- private volatile Server server;
- private AtomicBoolean initialized = new AtomicBoolean(false);
- private final BlockingQueue<HttpRequestContainer> containerQueue = new LinkedBlockingQueue<>(50);
+ private static final List<PropertyDescriptor> propertyDescriptors;
- @Override
- protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- final List<PropertyDescriptor> descriptors = new ArrayList<>();
+ static {
+ List<PropertyDescriptor> descriptors = new ArrayList<>();
descriptors.add(PORT);
descriptors.add(HOSTNAME);
descriptors.add(SSL_CONTEXT);
@@ -246,8 +247,17 @@ public class HandleHttpRequest extends AbstractProcessor {
descriptors.add(ALLOW_OPTIONS);
descriptors.add(ADDITIONAL_METHODS);
descriptors.add(CLIENT_AUTH);
+ descriptors.add(CONTAINER_QUEUE_SIZE);
+ propertyDescriptors = Collections.unmodifiableList(descriptors);
+ }
- return descriptors;
+ private volatile Server server;
+ private AtomicBoolean initialized = new AtomicBoolean(false);
+ private volatile BlockingQueue<HttpRequestContainer> containerQueue;
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return propertyDescriptors;
}
@Override
@@ -264,7 +274,7 @@ public class HandleHttpRequest extends AbstractProcessor {
if(initialized.get()){
return;
}
-
+ this.containerQueue = new LinkedBlockingQueue<>(context.getProperty(CONTAINER_QUEUE_SIZE).asInteger());
final String host = context.getProperty(HOSTNAME).getValue();
final int port = context.getProperty(PORT).asInteger();
final SSLContextService sslService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);
[26/27] nifi git commit: Updating appveyor to mimic Travis CI build.
Posted by al...@apache.org.
Updating appveyor to mimic Travis CI build.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/00d6feac
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/00d6feac
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/00d6feac
Branch: refs/heads/appveyor-improvement
Commit: 00d6feacd1e8f479e395263b1f1ee7abdfdee683
Parents: 220b768
Author: Aldrin Piri <al...@apache.org>
Authored: Fri Oct 14 09:22:06 2016 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Fri Oct 21 15:54:33 2016 -0400
----------------------------------------------------------------------
appveyor.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/00d6feac/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index c7aa2f2..d54984f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -32,9 +32,9 @@ install:
- cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g
- cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g
build_script:
- - mvn -q clean package --batch-mode -DskipTests
+ - mvn -q -T 2C -DskipTests clean install
test_script:
- - mvn -q clean install --batch-mode -Pcontrib-check
+ - mvn -P contrib-check verify
cache:
- C:\maven\
- C:\Users\appveyor\.m2
[11/27] nifi git commit: NIFI-2905: Log error stream of
ExecuteProcess cmd
Posted by al...@apache.org.
NIFI-2905: Log error stream of ExecuteProcess cmd
ExecuteProcess ignores error stream when Redirect Error Stream is
false, this commit let it to be logged instead so that user can see it on
bulletin.
This closes #1143.
Signed-off-by: Andy LoPresto <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/27dba60f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/27dba60f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/27dba60f
Branch: refs/heads/appveyor-improvement
Commit: 27dba60f27579ca9f33aa1afbf4bb46182d57df4
Parents: f287bc9
Author: Koji Kawamura <ij...@apache.org>
Authored: Mon Oct 17 17:07:32 2016 +0900
Committer: Andy LoPresto <al...@apache.org>
Committed: Tue Oct 18 15:56:27 2016 -0700
----------------------------------------------------------------------
.../processors/standard/ExecuteProcess.java | 3 +-
.../processors/standard/TestExecuteProcess.java | 46 ++++++++++++++++++++
2 files changed, 47 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/27dba60f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteProcess.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteProcess.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteProcess.java
index 62860ed..c0d3f2b 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteProcess.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteProcess.java
@@ -316,8 +316,7 @@ public class ExecuteProcess extends AbstractProcessor {
@Override
public void run() {
try (final BufferedReader reader = new BufferedReader(new InputStreamReader(externalProcess.getErrorStream()))) {
- while (reader.read() >= 0) {
- }
+ reader.lines().filter(line -> line != null && line.length() > 0).forEach(getLogger()::warn);
} catch (final IOException ioe) {
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/27dba60f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteProcess.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteProcess.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteProcess.java
index 160bbdb..8af82f1 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteProcess.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteProcess.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ExecutorService;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processors.standard.util.ArgumentUtils;
+import org.apache.nifi.util.LogMessage;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
@@ -205,4 +206,49 @@ public class TestExecuteProcess {
// assertEquals(inFile.length(), totalFlowFilesSize);
}
+
+ @Test
+ public void testNotRedirectErrorStream() {
+ final TestRunner runner = TestRunners.newTestRunner(ExecuteProcess.class);
+ runner.setProperty(ExecuteProcess.COMMAND, "cd");
+ runner.setProperty(ExecuteProcess.COMMAND_ARGUMENTS, "does-not-exist");
+
+ ProcessContext processContext = runner.getProcessContext();
+
+ ExecuteProcess processor = (ExecuteProcess) runner.getProcessor();
+ processor.updateScheduledTrue();
+ processor.setupExecutor(processContext);
+
+ processor.onTrigger(processContext, runner.getProcessSessionFactory());
+
+ final List<LogMessage> warnMessages = runner.getLogger().getWarnMessages();
+ assertEquals("If redirect error stream is false, " +
+ "the output should be logged as a warning so that user can notice on bulletin.", 1, warnMessages.size());
+ final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(ExecuteProcess.REL_SUCCESS);
+ assertEquals(0, succeeded.size());
+ }
+
+
+ @Test
+ public void testRedirectErrorStream() {
+ final TestRunner runner = TestRunners.newTestRunner(ExecuteProcess.class);
+ runner.setProperty(ExecuteProcess.COMMAND, "cd");
+ runner.setProperty(ExecuteProcess.COMMAND_ARGUMENTS, "does-not-exist");
+ runner.setProperty(ExecuteProcess.REDIRECT_ERROR_STREAM, "true");
+
+ ProcessContext processContext = runner.getProcessContext();
+
+ ExecuteProcess processor = (ExecuteProcess) runner.getProcessor();
+ processor.updateScheduledTrue();
+ processor.setupExecutor(processContext);
+
+ processor.onTrigger(processContext, runner.getProcessSessionFactory());
+
+ final List<LogMessage> warnMessages = runner.getLogger().getWarnMessages();
+ assertEquals("If redirect error stream is true " +
+ "the output should be sent as a content of flow-file.", 0, warnMessages.size());
+ final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(ExecuteProcess.REL_SUCCESS);
+ assertEquals(1, succeeded.size());
+ }
+
}
[16/27] nifi git commit: Dummy commit to clean stalled gh PRs
Posted by al...@apache.org.
Dummy commit to clean stalled gh PRs
This closes: #315 (superseded)
This closes: #1024 (superseded)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/231f5143
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/231f5143
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/231f5143
Branch: refs/heads/appveyor-improvement
Commit: 231f5143ab8100d6a5d4543a5476694378fdcf6d
Parents: ae503f5
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Thu Oct 20 23:02:33 2016 +1100
Committer: Andre F de Miranda <tr...@users.noreply.github.com>
Committed: Thu Oct 20 23:02:33 2016 +1100
----------------------------------------------------------------------
----------------------------------------------------------------------
[20/27] nifi git commit: NIFI-2685 - Configure surefire to use JAVA
Headless mode
Posted by al...@apache.org.
NIFI-2685 - Configure surefire to use JAVA Headless mode
This closes #961
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9f6fb617
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9f6fb617
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9f6fb617
Branch: refs/heads/appveyor-improvement
Commit: 9f6fb617ba440a77cc0e44dcf38ab0ce09a7a15c
Parents: f5f132b
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Fri Aug 26 17:04:17 2016 +1000
Committer: Matt Burgess <ma...@apache.org>
Committed: Thu Oct 20 16:46:54 2016 -0400
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9f6fb617/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7cf41b4..11bcd36 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1435,6 +1435,9 @@ language governing permissions and limitations under the License. -->
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18</version>
<configuration>
+ <systemPropertyVariables>
+ <java.awt.headless>true</java.awt.headless>
+ </systemPropertyVariables>
<includes>
<include>**/*Test.class</include>
<include>**/Test*.class</include>
[12/27] nifi git commit: NIFI-1710 - Resolve path name to nifi.sh
script
Posted by al...@apache.org.
NIFI-1710 - Resolve path name to nifi.sh script
This closes #1137.
Signed-off-by: Andy LoPresto <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/3914141c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/3914141c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/3914141c
Branch: refs/heads/appveyor-improvement
Commit: 3914141c45c574fb8d22a72af99165cb535f5f17
Parents: 27dba60
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Fri Oct 14 14:36:48 2016 +1100
Committer: Andy LoPresto <al...@apache.org>
Committed: Tue Oct 18 16:43:03 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/bin/nifi.sh | 28 ++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/3914141c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
index 79850ee..ff8addd 100755
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
@@ -17,12 +17,36 @@
# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches
-SCRIPT_DIR=$(dirname "$0")
-SCRIPT_NAME=$(basename "$0")
+# Discover the path of the file
+
+
+# Since MacOS X, FreeBSD and some other systems lack gnu readlink, we use a more portable
+# approach based on following StackOverflow comment http://stackoverflow.com/a/1116890/888876
+
+TARGET_FILE=$0
+
+cd $(dirname $TARGET_FILE)
+TARGET_FILE=$(basename $TARGET_FILE)
+
+# Iterate down a (possible) chain of symlinks
+while [ -L "$TARGET_FILE" ]
+do
+ TARGET_FILE=$(readlink $TARGET_FILE)
+ cd $(dirname $TARGET_FILE)
+ TARGET_FILE=$(basename $TARGET_FILE)
+done
+
+# Compute the canonicalized name by finding the physical path
+# for the directory we're in and appending the target file.
+PHYS_DIR=`pwd -P`
+
+SCRIPT_DIR=$PHYS_DIR
PROGNAME=$(basename "$0")
. "$SCRIPT_DIR"/nifi-env.sh
+
+
warn() {
echo "${PROGNAME}: $*"
}
[15/27] nifi git commit: Add jskora@apache.org to KEYS.
Posted by al...@apache.org.
Add jskora@apache.org to KEYS.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ae503f5d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ae503f5d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ae503f5d
Branch: refs/heads/appveyor-improvement
Commit: ae503f5d456a6170bd67cc9ce26c4eb6ed5c7673
Parents: 611cadd
Author: Joe Skora <js...@gmail.com>
Authored: Thu Oct 20 00:59:16 2016 -0400
Committer: Joe Skora <js...@gmail.com>
Committed: Thu Oct 20 00:59:16 2016 -0400
----------------------------------------------------------------------
KEYS | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/ae503f5d/KEYS
----------------------------------------------------------------------
diff --git a/KEYS b/KEYS
index e7b0e08..c2246d1 100644
--- a/KEYS
+++ b/KEYS
@@ -402,3 +402,61 @@ BzezmX5TWerF5OgeZCh5CtD/XRwubt7esaUGKcaQrMqV6YUU5CfgLMIk3GFHuvWR
XSQLfHE6JkMV7M6Y5ObosaDPZ0UAKl8XbZjwkc2Nf+fZoITI
=Gsga
-----END PGP PUBLIC KEY BLOCK-----
+pub 4096R/7D84EBD6 2016-10-13
+uid Joseph Skora (CODE SIGNING KEY) <js...@apache.org>
+sig 3 7D84EBD6 2016-10-13 Joseph Skora (CODE SIGNING KEY) <js...@apache.org>
+sub 4096R/CCFB335F 2016-10-13
+sig 7D84EBD6 2016-10-13 Joseph Skora (CODE SIGNING KEY) <js...@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFf+3HgBEADYhGAjqDg76ns9SinxM6GBWMbqkW0sFD77F9TqDEujajXtD7w2
+mAykzpxYpOBpCEQKzL8VDcpx/XaWD83/WAqZiP6K6LDjZWuY0HjCw03d5Cn5Vmzl
+92CAy5KLv0UqvF/xynT8yToW5OFC0FWdcQkc4z4pmyvPHMd2BNhpK3OfaRIr2zGN
+yBhsKm4guCv4A9cYtcba6AayJpggmJTowvgFee5mCsExVP9i7VogicS+1azfaPsO
+PVoJ4rvacbJ25qnBpblTkWwp3NFytmaaekORj6d3QK11ARfLs4AhviiZgW55BeU6
+k651wEnlsFju/vN1eaXQisS61lg1krVsLQKPd/cg/R4tRJNZti1lW744Zxnzstw0
+oMEa2ICVtDLLHNHM2TTHVODNGR3QwK2i+MQ1u+CSHyJZ2g5EybKnk19hkKQRjTdY
+2GU2c004kBCz3afmj5eVDo6z/1/Q5ehfGdfRcYIY/Qu0rbbOUxC5m4SmJUFQE0ud
+Hyijq0PRRUCr4vWyLZBDqf5Bh0SXHqAvoR8EMT2KECuQxoIFhogCOKpaEXF05xn8
+EFt+XqaE2puMnD9zIE2tYqTy/mbw2CPCRXFe5u+1GkqOQF7J6Lwx8v7o7L2bnwOe
+W05ZHur33aAYlQv4jd0cWMpvqbhGIjnv6EnyzH23xADPyvkIB8cJqGiXZwARAQAB
+tDNKb3NlcGggU2tvcmEgKENPREUgU0lHTklORyBLRVkpIDxqc2tvcmFAYXBhY2hl
+Lm9yZz6JAjcEEwECACECGwMCHgECF4AFAlf+5aQFCwkIBwMFFQoJCAsFFgIDAQAA
+CgkQlBwUQ32E69bZvhAAswWlZNptTgtlQo2AkRGB4junJsFPueuKR4D8FsNPio1A
+iegkd/gR9//Ld7NCxQaqHxf3lQ0lY6CS6Q/+Th7S/+/wNiCP7uPDa1uthrFbx3wg
+dz5aDakDOzwviliPlD2MFjehQIHHf0WwllzcTZXtHFVjsS45KABCNoT8F0tuFoqx
+nxHHMOhnFHMOsi4hFtJzD0tNvoZBsE9zZKZoobkWWki4YuLlR5gUOlyS91Is7kRg
+/YxODziWxxoNLp8MXCNVvokcSJTEnhZQMGqlpkoHpg9FcKdA4HVVHgtfeQXHVkud
+XHdD943zOUOYFZQMoFti8fe4W+8KxFeyByicWETUz5En98z0FlaK/wfYMCkWcZm4
+HwK8Xu4cJQrCl4bO9fJEqUBeCPW2BI2DArR7eAODUxXCJ/7Cd9m1zEsB7MY8X8cK
+mIeix/IjfgNyXlMQW8HLyDU521PaQQq9E0WLkhuQ2KbqSTDC70VaiILH7rpf6Hiu
+BMull3Nay/WMlK/sGc+/QXz9rA+vuMVvWTMX9uyBvMNI1bZDEUOHMiUFOKphWinR
+YJ9Qr9AX/w5Ew8y+Kwp7OZW+QSxZKAiWtjgBZqe8KytuYMUCWrBRjQTjb2SRCu+W
+aAV4+0D+xrc07LDqC9G2vG90OS6wYLjkvKq3siweawKjNss/avQY7zEpalrcpZG5
+Ag0EV/7ceAEQAKAsmYHVSR0q2FC/7TZNUB1HKB08c+mRBBb1rrkiB6WwobzJHmur
+8M0hSaOAHQsPVe9VMHkZfKLW3ohcBcn2IiRbwMaRcq4o/vACUzKzGL3BPpESksMG
+Rm5pSI7UPbZovLLSs1wN43FYjfz5OkN6DWbwXqQPj1vIEIlr+hu4XCJ+FitpA5P9
+FEGOgIHupllyfx3dJePatHvKnmkl9jQaqnIwSWvz1DOrG9RCBOLe174BePiVMaTt
+osEnLxsSsR/9PqbDvoH/jIrRWsfiZVd8kFRx5VmRlrHFUBxlQDH/zhrjVsH9TRFI
+PKOo4XpeB/QS9TNdBe6nHEwYjIa0Bg/Ckb7BtCOs5Pcz5xY6Q3JRWc7PtuQAG3mj
+tC8cc5wvK+yWPQFQHUTgB7McNmTg2CP6DttcZGnt+5+WUm+4kT4KnCYUMK301qBT
+7cltRWKHuPrnDZQU0qp8pjGQxiRWWh2EQa5Jf4+Sww6Wn8YKLRhEsyDZg/DpvtrH
+CSAU2XV5Ma8MjGNohz7zzY003ket0ywG401u8IQ482L4PwF/Ya/kRPLWqTr1jYQ/
+laGGfVBgIBxqgqmAdL0y31QHsWZvsg8a+QHuFQXFaRZXXhjyDZsdwcoCqfHJEGuF
+murGfiIBVwTgEn1llnJm1XyYfJ/VHzbarTOSE5FGN2pIgNX6wlzCuJiXABEBAAGJ
+Ah8EGAECAAkFAlf+3HgCGwwACgkQlBwUQ32E69Yo5Q/+O6Rh8w8fDZp6Ix87lhW7
+AyAeAnKJmKIpXHSAfDwznzrsptjMPsYWIdKZh/Oj7qkR/hx/dErK9m6qKZm5rGiy
+b8JhvxYrgyozqqYC1ueYow/POaUeVUQNYGOc0tf+FFVBh89ml7+hOBzZjauEdXlj
+0QTwjR39KYpWo6fudusZWRVv8thOlV/DQFMBa0uPQsivlb0P7yBqrkzQi7aHKfLo
+rAhRpKtqsN3Uv3dpGrcBIwwMfobxwyTlYTPqQckv8nsTIVSGqYRZMSPRUDoNQgkt
+6PW9Zn73wdqmQtJ2PekKz8jQMCxXxrbASF0UUfVaTt1ezmjqP7BiThGX3VQqcZM8
+qUb3l2nMSEhG26qln9pxHH5UMfVGVa0euPuKq/7eZZoKDwm1kw0gxByEV/g9Bm/6
+FY3WOK3fa3Z7otcZ/yt/EbU2/6qCMFactRXbJqH1gb1sdWIfK9PIysLJld3K2nIw
+QrSl3kGaEBuL2gXBQdMTwliBb56gI8P+K3W9kgQCOR7m4w1UpT+OaJ9J0FriNz+b
+pPDMQJgdQkaoj80wsIi7kWCy6lt2GuHbOLKT8CX0MfVCTYlCb4ubgCNPjru3eVWo
+52ujPZY/uzwR2STGx3wf2n/dNSHXBsWybLo6agvSEhjCuT1dh/lmPCg1pZvlmKmq
+Ry2jgybWqDPTomJ0jsgp7GA=
+=h5+w
+-----END PGP PUBLIC KEY BLOCK-----
[04/27] nifi git commit: NIFI-2901: Comment out hadoop.version
overrides for vendor profiles
Posted by al...@apache.org.
NIFI-2901: Comment out hadoop.version overrides for vendor profiles
This closes #1139
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d63e675a
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d63e675a
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d63e675a
Branch: refs/heads/appveyor-improvement
Commit: d63e675a7d2905ae8408adc8114c01c2ea4604c8
Parents: b52b839
Author: Matt Burgess <ma...@apache.org>
Authored: Fri Oct 14 12:05:52 2016 -0400
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Fri Oct 14 12:40:31 2016 -0400
----------------------------------------------------------------------
pom.xml | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d63e675a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 497823b..735a9ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1869,8 +1869,9 @@ language governing permissions and limitations under the License. -->
</repositories>
<properties>
<!-- Vendor-specific version number included here as default, should be overridden on the
- command-line -->
+ command-line
<hadoop.version>2.7.1.2.4.0.0-169</hadoop.version>
+ -->
</properties>
</profile>
<profile>
@@ -1892,8 +1893,9 @@ language governing permissions and limitations under the License. -->
</repositories>
<properties>
<!-- Vendor-specific version number included here as default, should be overridden on the
- command-line -->
+ command-line
<hadoop.version>2.7.0-mapr-1602</hadoop.version>
+ -->
</properties>
</profile>
<profile>
@@ -1915,8 +1917,9 @@ language governing permissions and limitations under the License. -->
</repositories>
<properties>
<!-- Vendor-specific version number included here as default, should be overridden on the
- command-line -->
+ command-line
<hadoop.version>2.6.0-cdh5.8.1</hadoop.version>
+ -->
</properties>
</profile>
</profiles>
[24/27] nifi git commit: Remove repo cleanup in favor of delegation
to JUnit TemporaryFolder.
Posted by al...@apache.org.
Remove repo cleanup in favor of delegation to JUnit TemporaryFolder.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/81313dda
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/81313dda
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/81313dda
Branch: refs/heads/appveyor-improvement
Commit: 81313dda6cfa9708aad70c653fc96646eeeb89fd
Parents: c49243e
Author: Aldrin Piri <al...@apache.org>
Authored: Fri Oct 14 13:37:26 2016 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Fri Oct 21 15:54:33 2016 -0400
----------------------------------------------------------------------
.../TestPersistentProvenanceRepository.java | 24 --------------------
1 file changed, 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/81313dda/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
index 4a0cd84..0fcbacd 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
@@ -146,30 +146,6 @@ public class TestPersistentProvenanceRepository {
} catch (final IOException ioe) {
}
}
-
- // Delete all of the storage files. We do this in order to clean up the tons of files that
- // we create but also to ensure that we have closed all of the file handles. If we leave any
- // streams open, for instance, this will throw an IOException, causing our unit test to fail.
- for (final File storageDir : config.getStorageDirectories()) {
- int i;
- for (i = 0; i < 3; i++) {
- try {
- FileUtils.deleteFile(storageDir, true);
- break;
- } catch (final IOException ioe) {
- // if there is a virus scanner, etc. running in the background we may not be able to
- // delete the file. Wait a sec and try again.
- if (i == 2) {
- throw ioe;
- } else {
- try {
- Thread.sleep(1000L);
- } catch (final InterruptedException ie) {
- }
- }
- }
- }
- }
}
[25/27] nifi git commit: Making the TemporaryFolder public.
Posted by al...@apache.org.
Making the TemporaryFolder public.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c49243ee
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c49243ee
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c49243ee
Branch: refs/heads/appveyor-improvement
Commit: c49243eec970781244eb0555b1395a128c9af0d4
Parents: df61cde
Author: Aldrin Piri <al...@apache.org>
Authored: Fri Oct 14 12:09:31 2016 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Fri Oct 21 15:54:33 2016 -0400
----------------------------------------------------------------------
.../apache/nifi/provenance/TestPersistentProvenanceRepository.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/c49243ee/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
index 7c8a6dc..4a0cd84 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
@@ -99,7 +99,7 @@ public class TestPersistentProvenanceRepository {
public TestName name = new TestName();
@Rule
- TemporaryFolder testFolder = new TemporaryFolder();
+ public TemporaryFolder testFolder = new TemporaryFolder();
private PersistentProvenanceRepository repo;
private RepositoryConfiguration config;
[22/27] nifi git commit: Dummy commit to clean stalled gh PRs
Posted by al...@apache.org.
Dummy commit to clean stalled gh PRs
This closes: #513 (superseded)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/220b7680
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/220b7680
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/220b7680
Branch: refs/heads/appveyor-improvement
Commit: 220b76803b2f70ddd6dcc366478d722599349983
Parents: b864d49
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Thu Oct 20 23:58:39 2016 +1100
Committer: Andre F de Miranda <tr...@users.noreply.github.com>
Committed: Fri Oct 21 23:17:44 2016 +1100
----------------------------------------------------------------------
----------------------------------------------------------------------