You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2018/05/10 15:49:53 UTC
[01/50] nifi git commit: NIFI-4428: Replaced JSON input with
RecordReader using per-record flowfiles NIFI-4428: Added initial L&N,
fixed dependency hierarchy
Repository: nifi
Updated Branches:
refs/heads/HDF-3.1-maint [created] 29b82727a
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/processors/druid/PutDruidRecordTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/processors/druid/PutDruidRecordTest.java b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/processors/druid/PutDruidRecordTest.java
new file mode 100644
index 0000000..28e084c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/processors/druid/PutDruidRecordTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.druid;
+
+
+import org.apache.nifi.controller.druid.DruidTranquilityController;
+import org.apache.nifi.controller.druid.MockDruidTranquilityController;
+import org.apache.nifi.controller.api.druid.DruidTranquilityService;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.serialization.record.MockRecordParser;
+import org.apache.nifi.serialization.record.MockRecordWriter;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class PutDruidRecordTest {
+
+ private TestRunner runner;
+ private DruidTranquilityService druidTranquilityController;
+ private MockRecordParser recordReader;
+ private MockRecordWriter recordWriter;
+
+ @Before
+ public void setUp() throws Exception {
+ runner = TestRunners.newTestRunner(PutDruidRecord.class);
+ druidTranquilityController = new MockDruidTranquilityController(2,3);
+ recordReader = new MockRecordParser();
+ recordWriter = new MockRecordWriter(null, true, 2);
+ runner.addControllerService("reader", recordReader);
+ runner.enableControllerService(recordReader);
+ runner.addControllerService("writer", recordWriter);
+ runner.enableControllerService(recordWriter);
+
+ runner.addControllerService("tranquility", druidTranquilityController);
+ runner.setProperty(druidTranquilityController, DruidTranquilityController.DATASOURCE, "test");
+ runner.setProperty(druidTranquilityController, DruidTranquilityController.ZOOKEEPER_CONNECTION_STRING, "localhost:2181");
+ runner.setProperty(druidTranquilityController, DruidTranquilityController.AGGREGATOR_JSON, "[{\"type\": \"count\", \"name\": \"count\"}]");
+ runner.setProperty(druidTranquilityController, DruidTranquilityController.DIMENSIONS_LIST, "dim1,dim2");
+ runner.assertValid(druidTranquilityController);
+ runner.enableControllerService(druidTranquilityController);
+
+ runner.setProperty(PutDruidRecord.RECORD_READER_FACTORY, "reader");
+ runner.setProperty(PutDruidRecord.RECORD_WRITER_FACTORY, "writer");
+ runner.setProperty(PutDruidRecord.DRUID_TRANQUILITY_SERVICE, "tranquility");
+ }
+
+ @Test
+ public void testEmptyRecord() throws Exception {
+ recordReader.addSchemaField("name", RecordFieldType.STRING);
+ recordReader.addSchemaField("age", RecordFieldType.INT);
+ recordReader.addSchemaField("sport", RecordFieldType.STRING);
+
+ runner.enqueue("");
+ runner.run();
+
+ runner.assertTransferCount(PutDruidRecord.REL_SUCCESS, 1);
+ runner.assertTransferCount(PutDruidRecord.REL_FAILURE, 0);
+ runner.assertTransferCount(PutDruidRecord.REL_DROPPED, 0);
+ MockFlowFile flowFile = runner.getFlowFilesForRelationship(PutDruidRecord.REL_SUCCESS).get(0);
+ flowFile.assertAttributeEquals(PutDruidRecord.RECORD_COUNT, "0");
+ }
+
+ @Test
+ public void testPutRecords() throws Exception {
+ recordReader.addSchemaField("name", RecordFieldType.STRING);
+ recordReader.addSchemaField("age", RecordFieldType.INT);
+ recordReader.addSchemaField("sport", RecordFieldType.STRING);
+
+ recordReader.addRecord("John Doe", 48, "Soccer");
+ recordReader.addRecord("Jane Doe", 47, "Tennis");
+ recordReader.addRecord("Sally Doe", 47, "Curling"); // Will be dropped due to the "drop after 2" parameter on the MockDruidTranquilityController
+ recordReader.addRecord("Jimmy Doe", 14, null); // Will fail due to the "fail after 3" parameter on the MockDruidTranquilityController
+ recordReader.addRecord("Pizza Doe", 14, null); // Will fail due to the "fail after 3" parameter on the MockDruidTranquilityController
+ recordReader.addRecord("Bad Record", "X", 13); // RecordWriter fail due to the "fail after 2" parameter on the MockRecordWriter, not written to output
+
+ runner.enqueue("");
+ runner.run();
+
+ runner.assertTransferCount(PutDruidRecord.REL_SUCCESS, 1);
+ runner.assertTransferCount(PutDruidRecord.REL_FAILURE, 1);
+ runner.assertTransferCount(PutDruidRecord.REL_DROPPED, 1);
+ MockFlowFile flowFile = runner.getFlowFilesForRelationship(PutDruidRecord.REL_SUCCESS).get(0);
+ flowFile.assertAttributeEquals(PutDruidRecord.RECORD_COUNT, "2");
+ flowFile = runner.getFlowFilesForRelationship(PutDruidRecord.REL_DROPPED).get(0);
+ flowFile.assertAttributeEquals(PutDruidRecord.RECORD_COUNT, "1");
+ flowFile = runner.getFlowFilesForRelationship(PutDruidRecord.REL_FAILURE).get(0);
+ flowFile.assertAttributeEquals(PutDruidRecord.RECORD_COUNT, "2");
+
+ // Assert a single SEND event present for the successful flow file
+ assertEquals(1, runner.getProvenanceEvents().stream().filter((e) -> ProvenanceEventType.SEND.equals(e.getEventType())).count());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/pom.xml
new file mode 100644
index 0000000..82d3017
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/pom.xml
@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-druid-bundle</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <properties>
+ <druid.version>0.9.1</druid.version>
+ <tranquility.version>0.8.2</tranquility.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>tranquility-core_2.11</artifactId>
+ <version>${tranquility.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>druid-processing</artifactId>
+ <version>${druid.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <modules>
+ <module>nifi-druid-nar</module>
+ <module>nifi-druid-controller-service-api-nar</module>
+ <module>nifi-druid-controller-service-api</module>
+ <module>nifi-druid-controller-service</module>
+ <module>nifi-druid-processors</module>
+ </modules>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml
index 70a5e3c..0812c67 100755
--- a/nifi-nar-bundles/pom.xml
+++ b/nifi-nar-bundles/pom.xml
@@ -91,6 +91,7 @@
<module>nifi-metrics-reporting-bundle</module>
<module>nifi-spark-bundle</module>
<module>nifi-atlas-bundle</module>
+ <module>nifi-druid-bundle</module>
</modules>
<build>
[40/50] nifi git commit: NIFI-4969 Fixing error when importing
versioned flow with a processor that uses event driven scheduling. This
closes #2539
Posted by al...@apache.org.
NIFI-4969 Fixing error when importing versioned flow with a processor that uses event driven scheduling. This closes #2539
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/79e4402e
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/79e4402e
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/79e4402e
Branch: refs/heads/HDF-3.1-maint
Commit: 79e4402e22fbbdd68418f6257da2637f82c75aa3
Parents: 4adb5b7
Author: Bryan Bende <bb...@apache.org>
Authored: Tue Mar 13 12:42:33 2018 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Mar 13 13:33:50 2018 -0400
----------------------------------------------------------------------
.../java/org/apache/nifi/controller/StandardProcessorNode.java | 3 ++-
.../main/java/org/apache/nifi/groups/StandardProcessGroup.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/79e4402e/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
index 187b62f..4ce9420 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
@@ -621,7 +621,8 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
throw new IllegalStateException("Cannot modify Processor configuration while the Processor is running");
}
if (taskCount < 1 && getSchedulingStrategy() != SchedulingStrategy.EVENT_DRIVEN) {
- throw new IllegalArgumentException();
+ throw new IllegalArgumentException("Cannot set Concurrent Tasks to " + taskCount + " for component "
+ + getIdentifier() + " because Scheduling Strategy is not Event Driven");
}
if (!isTriggeredSerially()) {
concurrentTaskCount.set(taskCount);
http://git-wip-us.apache.org/repos/asf/nifi/blob/79e4402e/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index 4177917..f0b7b3e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -4072,8 +4072,6 @@ public final class StandardProcessGroup implements ProcessGroup {
processor.setAnnotationData(proposed.getAnnotationData());
processor.setBulletinLevel(LogLevel.valueOf(proposed.getBulletinLevel()));
processor.setComments(proposed.getComments());
- processor.setMaxConcurrentTasks(proposed.getConcurrentlySchedulableTaskCount());
- processor.setExecutionNode(ExecutionNode.valueOf(proposed.getExecutionNode()));
processor.setName(proposed.getName());
processor.setPenalizationPeriod(proposed.getPenaltyDuration());
@@ -4082,6 +4080,8 @@ public final class StandardProcessGroup implements ProcessGroup {
processor.setRunDuration(proposed.getRunDurationMillis(), TimeUnit.MILLISECONDS);
processor.setSchedulingStrategy(SchedulingStrategy.valueOf(proposed.getSchedulingStrategy()));
processor.setScheduldingPeriod(proposed.getSchedulingPeriod());
+ processor.setMaxConcurrentTasks(proposed.getConcurrentlySchedulableTaskCount());
+ processor.setExecutionNode(ExecutionNode.valueOf(proposed.getExecutionNode()));
processor.setStyle(proposed.getStyle());
processor.setYieldPeriod(proposed.getYieldDuration());
processor.setPosition(new Position(proposed.getPosition().getX(), proposed.getPosition().getY()));
[09/50] nifi git commit: NIFI-4767 - Fixed issues with RecordPath
using the wrong field name for arrays and maps. Also addressed issue where
Avro Reader was returning a Record object when it should return a Map
Posted by al...@apache.org.
NIFI-4767 - Fixed issues with RecordPath using the wrong field name for arrays and maps. Also addressed issue where Avro Reader was returning a Record object when it should return a Map
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #2398.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ad0b160e
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ad0b160e
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ad0b160e
Branch: refs/heads/HDF-3.1-maint
Commit: ad0b160e0dc4565a5f98e885dd1c7e56b0ee32d7
Parents: 9ffdeb7
Author: Mark Payne <ma...@hotmail.com>
Authored: Thu Jan 11 15:04:14 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 14:58:22 2018 -0500
----------------------------------------------------------------------
.../record/path/paths/MultiArrayIndexPath.java | 16 ++--
.../nifi/record/path/paths/MultiMapKeyPath.java | 2 +-
.../record/path/paths/SingularMapKeyPath.java | 2 +-
.../record/path/paths/WildcardIndexPath.java | 2 +-
.../apache/nifi/record/path/TestRecordPath.java | 79 ++++++++++++++++----
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 8 +-
6 files changed, 75 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad0b160e/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
index 88ce614..ce680c6 100644
--- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
@@ -19,7 +19,6 @@ package org.apache.nifi.record.path.paths;
import java.util.ArrayList;
import java.util.List;
-import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.nifi.record.path.ArrayIndexFieldValue;
@@ -56,24 +55,19 @@ public class MultiArrayIndexPath extends RecordPathSegment {
return indices.stream()
.filter(range -> values.length > Math.abs(range.getMin()))
.flatMap(range -> {
- final List<Object> valuesWithinRange = new ArrayList<>();
- final List<Integer> indexes = new ArrayList<Integer>();
-
final int min = range.getMin() < 0 ? values.length + range.getMin() : range.getMin();
final int max = range.getMax() < 0 ? values.length + range.getMax() : range.getMax();
+ final List<FieldValue> indexFieldValues = new ArrayList<>(Math.max(0, max - min));
for (int i = min; i <= max; i++) {
if (values.length > i) {
- valuesWithinRange.add(values[i]);
- indexes.add(i);
+ final RecordField elementField = new RecordField(arrayField.getFieldName(), elementDataType);
+ final FieldValue arrayIndexFieldValue = new ArrayIndexFieldValue(values[i], elementField, fieldValue, i);
+ indexFieldValues.add(arrayIndexFieldValue);
}
}
- return IntStream.range(0, valuesWithinRange.size())
- .mapToObj(index -> {
- final RecordField elementField = new RecordField(arrayField.getFieldName(), elementDataType);
- return new ArrayIndexFieldValue(valuesWithinRange.get(index), elementField, fieldValue, indexes.get(index));
- });
+ return indexFieldValues.stream();
});
});
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad0b160e/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
index e9b1874..dfb7071 100644
--- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
@@ -49,7 +49,7 @@ public class MultiMapKeyPath extends RecordPathSegment {
final Map<String, ?> map = (Map<String, ?>) fieldValue.getValue();
return mapKeys.stream().map(key -> {
final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
- final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "['" + key + "']", valueType);
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType);
return new MapEntryFieldValue(map.get(key), elementField, fieldValue, key);
});
});
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad0b160e/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
index ee57f36..201ddc1 100644
--- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
@@ -45,7 +45,7 @@ public class SingularMapKeyPath extends RecordPathSegment {
.filter(Filters.fieldTypeFilter(RecordFieldType.MAP))
.map(fieldValue -> {
final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
- final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "['" + mapKey + "']", valueType);
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType);
return new MapEntryFieldValue(getMapValue(fieldValue), elementField, fieldValue, mapKey);
});
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad0b160e/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
index b9241e8..ab4b176 100644
--- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
@@ -59,7 +59,7 @@ public class WildcardIndexPath extends RecordPathSegment {
return map.entrySet().stream()
.map(entry -> {
final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
- final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "['" + entry.getKey() + "']", valueType);
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType);
return new MapEntryFieldValue(entry.getValue(), elementField, fieldValue, entry.getKey());
});
} else {
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad0b160e/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
index 0c96111..67e390a 100644
--- a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
+++ b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
@@ -17,6 +17,7 @@
package org.apache.nifi.record.path;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -28,7 +29,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.nifi.record.path.exception.RecordPathException;
@@ -134,6 +134,10 @@ public class TestRecordPath {
assertEquals("balance", fieldValues.get(1).getField().getFieldName());
assertEquals(123.45D, fieldValues.get(1).getValue());
+
+ RecordPath.compile("/mainAccount/*[. > 100]").evaluate(record).getSelectedFields().forEach(field -> field.updateValue(122.44D));
+ assertEquals(1, accountValues.get("id"));
+ assertEquals(122.44D, accountValues.get("balance"));
}
@Test
@@ -154,9 +158,21 @@ public class TestRecordPath {
assertEquals(1, fieldValues.size());
final FieldValue fieldValue = fieldValues.get(0);
- assertTrue(fieldValue.getField().getFieldName().equals("accounts"));
+ assertEquals("accounts", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
assertEquals(accountRecord, fieldValue.getValue());
+
+ final Map<String, Object> updatedAccountValues = new HashMap<>(accountValues);
+ updatedAccountValues.put("balance", 122.44D);
+ final Record updatedAccountRecord = new MapRecord(getAccountSchema(), updatedAccountValues);
+ RecordPath.compile("/*[0]").evaluate(record).getSelectedFields().forEach(field -> field.updateValue(updatedAccountRecord));
+
+ final Object[] accountRecords = (Object[]) record.getValue("accounts");
+ assertEquals(1, accountRecords.length);
+ final Record recordToVerify = (Record) accountRecords[0];
+ assertEquals(122.44D, recordToVerify.getValue("balance"));
+ assertEquals(48, record.getValue("id"));
+ assertEquals("John Doe", record.getValue("name"));
}
@Test
@@ -228,12 +244,31 @@ public class TestRecordPath {
final Record record = new MapRecord(schema, values);
final FieldValue fieldValue = RecordPath.compile("/attributes['city']").evaluate(record).getSelectedFields().findFirst().get();
- assertTrue(fieldValue.getField().getFieldName().startsWith("attributes['"));
+ assertTrue(fieldValue.getField().getFieldName().equals("attributes"));
assertEquals("New York", fieldValue.getValue());
assertEquals(record, fieldValue.getParentRecord().get());
}
@Test
+ @SuppressWarnings("unchecked")
+ public void testUpdateMap() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("city", "New York");
+ attributes.put("state", "NY");
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("attributes", attributes);
+ final Record record = new MapRecord(schema, values);
+
+ RecordPath.compile("/attributes['city']").evaluate(record).getSelectedFields().findFirst().get().updateValue("Boston");
+ assertEquals("Boston", ((Map<String, Object>) record.getValue("attributes")).get("city"));
+ }
+
+ @Test
public void testMapWildcard() {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
@@ -254,9 +289,18 @@ public class TestRecordPath {
assertEquals("NY", fieldValues.get(1).getValue());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().startsWith("attributes['"));
+ assertEquals("attributes", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
+
+ RecordPath.compile("/attributes[*]").evaluate(record).getSelectedFields().forEach(field -> field.updateValue("Unknown"));
+ assertEquals("Unknown", attributes.get("city"));
+ assertEquals("Unknown", attributes.get("state"));
+
+ RecordPath.compile("/attributes[*][fieldName(.) = 'attributes']").evaluate(record).getSelectedFields().forEach(field -> field.updateValue("Unknown"));
+ assertEquals("Unknown", attributes.get("city"));
+ assertEquals("Unknown", attributes.get("state"));
+
}
@Test
@@ -280,9 +324,13 @@ public class TestRecordPath {
assertEquals("NY", fieldValues.get(1).getValue());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().startsWith("attributes['"));
+ assertEquals("attributes", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
+
+ RecordPath.compile("/attributes['city', 'state']").evaluate(record).getSelectedFields().forEach(field -> field.updateValue("Unknown"));
+ assertEquals("Unknown", attributes.get("city"));
+ assertEquals("Unknown", attributes.get("state"));
}
@Test
@@ -314,7 +362,7 @@ public class TestRecordPath {
final Record record = new MapRecord(schema, values);
final FieldValue fieldValue = RecordPath.compile("/numbers[3]").evaluate(record).getSelectedFields().findFirst().get();
- assertTrue(fieldValue.getField().getFieldName().equals("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(3, fieldValue.getValue());
assertEquals(record, fieldValue.getParentRecord().get());
}
@@ -330,7 +378,7 @@ public class TestRecordPath {
final List<FieldValue> fieldValues = RecordPath.compile("/numbers[0..1]").evaluate(record).getSelectedFields().collect(Collectors.toList());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().equals("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
@@ -354,11 +402,13 @@ public class TestRecordPath {
int i = 0;
final int[] expectedValues = new int[] {3, 6, 9, 8};
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().startsWith("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(expectedValues[i++], fieldValue.getValue());
assertEquals(record, fieldValue.getParentRecord().get());
}
+ RecordPath.compile("/numbers[3,6, -1, -2]").evaluate(record).getSelectedFields().forEach(field -> field.updateValue(99));
+ assertArrayEquals(new Object[] {0, 1, 2, 99, 4, 5, 99, 7, 99, 99}, (Object[]) values.get("numbers"));
}
@Test
@@ -372,7 +422,7 @@ public class TestRecordPath {
List<FieldValue> fieldValues = RecordPath.compile("/numbers[0, 2, 4..7, 9]").evaluate(record).getSelectedFields().collect(Collectors.toList());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().startsWith("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
@@ -384,7 +434,7 @@ public class TestRecordPath {
fieldValues = RecordPath.compile("/numbers[0..-1]").evaluate(record).getSelectedFields().collect(Collectors.toList());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().equals("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
expectedValues = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
@@ -396,7 +446,7 @@ public class TestRecordPath {
fieldValues = RecordPath.compile("/numbers[-1..-1]").evaluate(record).getSelectedFields().collect(Collectors.toList());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().equals("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
expectedValues = new int[] {9};
@@ -407,7 +457,7 @@ public class TestRecordPath {
fieldValues = RecordPath.compile("/numbers[*]").evaluate(record).getSelectedFields().collect(Collectors.toList());
for (final FieldValue fieldValue : fieldValues) {
- assertTrue(fieldValue.getField().getFieldName().equals("numbers"));
+ assertEquals("numbers", fieldValue.getField().getFieldName());
assertEquals(record, fieldValue.getParentRecord().get());
}
expectedValues = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
@@ -441,7 +491,7 @@ public class TestRecordPath {
for (final FieldValue fieldValue : fieldValues) {
final String fieldName = fieldValue.getField().getFieldName();
- assertTrue(Pattern.compile("numbers").matcher(fieldName).matches());
+ assertEquals("numbers", fieldName);
assertEquals(RecordFieldType.INT, fieldValue.getField().getDataType().getFieldType());
assertEquals(4, fieldValue.getValue());
assertEquals(record, fieldValue.getParentRecord().get());
@@ -478,6 +528,9 @@ public class TestRecordPath {
assertEquals(accountRecord, fieldValue.getParentRecord().get());
assertEquals(123.45D, fieldValue.getValue());
assertEquals("balance", fieldValue.getField().getFieldName());
+
+ RecordPath.compile("/mainAccount/././balance/.").evaluate(record).getSelectedFields().forEach(field -> field.updateValue(123.44D));
+ assertEquals(123.44D, accountValues.get("balance"));
}
@Test
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad0b160e/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index cc1fd38..5bdd70b 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -822,13 +822,7 @@ public class AvroTypeUtil {
map.put(key, obj);
}
- final DataType elementType = AvroTypeUtil.determineDataType(avroSchema.getValueType());
- final List<RecordField> mapFields = new ArrayList<>();
- for (final String key : map.keySet()) {
- mapFields.add(new RecordField(key, elementType, true));
- }
- final RecordSchema mapSchema = new SimpleRecordSchema(mapFields);
- return new MapRecord(mapSchema, map);
+ return map;
}
return value;
[28/50] nifi git commit: NIFI-4828: Fix MergeContent to process all
ready bins
Posted by al...@apache.org.
NIFI-4828: Fix MergeContent to process all ready bins
Before this fix, MergeContent only processed the first bin even if there
were multiple bins.
There were two unit tests marked with Ignore those had been
failing because of this.
This closes #2444.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/63098baa
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/63098baa
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/63098baa
Branch: refs/heads/HDF-3.1-maint
Commit: 63098baa11f4048469413e15d3e7bdf27857b369
Parents: 678dfc1
Author: Koji Kawamura <ij...@apache.org>
Authored: Tue Jan 30 15:21:00 2018 +0900
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Feb 7 15:50:33 2018 -0500
----------------------------------------------------------------------
.../nifi/processor/util/bin/BinFiles.java | 60 ++++++++++----------
.../nifi/processor/util/bin/BinManager.java | 47 ++++++---------
.../processors/standard/TestMergeContent.java | 7 +--
3 files changed, 47 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/63098baa/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
index 7f79b70..643aae4 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
@@ -217,48 +217,45 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
}
private int processBins(final ProcessContext context) {
- final Bin bin = readyBins.poll();
- if (bin == null) {
- return 0;
- }
-
- final List<Bin> bins = new ArrayList<>();
- bins.add(bin);
-
final ComponentLog logger = getLogger();
+ int processedBins = 0;
+ Bin bin;
+ while ((bin = readyBins.poll()) != null) {
+ boolean binAlreadyCommitted;
+ try {
+ binAlreadyCommitted = this.processBin(bin, context);
+ } catch (final ProcessException e) {
+ logger.error("Failed to process bundle of {} files due to {}", new Object[] {bin.getContents().size(), e});
+
+ final ProcessSession binSession = bin.getSession();
+ for (final FlowFile flowFile : bin.getContents()) {
+ binSession.transfer(flowFile, REL_FAILURE);
+ }
+ binSession.commit();
+ continue;
+ } catch (final Exception e) {
+ logger.error("Failed to process bundle of {} files due to {}; rolling back sessions", new Object[] {bin.getContents().size(), e});
- boolean binAlreadyCommitted = false;
- try {
- binAlreadyCommitted = this.processBin(bin, context);
- } catch (final ProcessException e) {
- logger.error("Failed to process bundle of {} files due to {}", new Object[] {bin.getContents().size(), e});
-
- final ProcessSession binSession = bin.getSession();
- for (final FlowFile flowFile : bin.getContents()) {
- binSession.transfer(flowFile, REL_FAILURE);
+ bin.getSession().rollback();
+ continue;
}
- binSession.commit();
- return 1;
- } catch (final Exception e) {
- logger.error("Failed to process bundle of {} files due to {}; rolling back sessions", new Object[] {bin.getContents().size(), e});
- bin.getSession().rollback();
- return 1;
- }
+ // If this bin's session has been committed, move on.
+ if (!binAlreadyCommitted) {
+ final ProcessSession binSession = bin.getSession();
+ binSession.transfer(bin.getContents(), REL_ORIGINAL);
+ binSession.commit();
+ }
- // If this bin's session has been committed, move on.
- if (!binAlreadyCommitted) {
- final ProcessSession binSession = bin.getSession();
- binSession.transfer(bin.getContents(), REL_ORIGINAL);
- binSession.commit();
+ processedBins++;
}
- return 1;
+ return processedBins;
}
private int binFlowFiles(final ProcessContext context, final ProcessSessionFactory sessionFactory) {
int flowFilesBinned = 0;
- while (binManager.getBinCount() <= context.getProperty(MAX_BIN_COUNT).asInteger().intValue()) {
+ while (binManager.getBinCount() <= context.getProperty(MAX_BIN_COUNT).asInteger()) {
if (!isScheduled()) {
break;
}
@@ -290,6 +287,7 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
bin.offer(flowFile, session);
this.readyBins.add(bin);
}
+ flowFilesBinned += entry.getValue().size();
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/63098baa/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinManager.java b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinManager.java
index d6a8567..e6cec78 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinManager.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/bin/BinManager.java
@@ -175,40 +175,25 @@ public class BinManager {
continue;
}
- final List<Bin> currentBins = groupBinMap.get(groupIdentifier);
- if (currentBins == null) { // this is a new group we need to register
- final List<Bin> bins = new ArrayList<>();
- final Bin bin = new Bin(sessionFactory.createSession(), minSizeBytes.get(), currentMaxSizeBytes, minEntries.get(),
- maxEntries.get(), fileCountAttribute.get());
- bins.add(bin);
- groupBinMap.put(groupIdentifier, bins);
- binCount++;
-
- final boolean added = bin.offer(flowFile, session);
- if (!added) {
- unbinned.add(flowFile);
- }
- continue;
- } else {
- for (final Bin bin : currentBins) {
- final boolean accepted = bin.offer(flowFile, session);
- if (accepted) {
- continue flowFileLoop;
- }
- }
-
- //if we've reached this point then we couldn't fit it into any existing bins - gotta make a new one
- final Bin bin = new Bin(sessionFactory.createSession(), minSizeBytes.get(), currentMaxSizeBytes, minEntries.get(),
- maxEntries.get(), fileCountAttribute.get());
- currentBins.add(bin);
- binCount++;
- final boolean added = bin.offer(flowFile, session);
- if (!added) {
- unbinned.add(flowFile);
+ final List<Bin> currentBins = groupBinMap.computeIfAbsent(groupIdentifier, k -> new ArrayList<>());
+ for (final Bin bin : currentBins) {
+ final boolean accepted = bin.offer(flowFile, session);
+ if (accepted) {
+ continue flowFileLoop;
}
+ }
- continue;
+ // if we've reached this point then the groupIdentifier was a brand new one,
+ // or we couldn't fit it into any existing bins - gotta make a new one
+ final Bin bin = new Bin(sessionFactory.createSession(), minSizeBytes.get(), currentMaxSizeBytes, minEntries.get(),
+ maxEntries.get(), fileCountAttribute.get());
+ currentBins.add(bin);
+ binCount++;
+ final boolean added = bin.offer(flowFile, session);
+ if (!added) {
+ unbinned.add(flowFile);
}
+
}
} finally {
wLock.unlock();
http://git-wip-us.apache.org/repos/asf/nifi/blob/63098baa/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
index d461712..3d8e015 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
@@ -57,7 +57,6 @@ import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
public class TestMergeContent {
@@ -911,7 +910,6 @@ public class TestMergeContent {
assembled.assertContentEquals("A Man A Plan A Canal Panama".getBytes("UTF-8"));
}
- @Ignore("this test appears to be faulty")
@Test
public void testDefragmentMultipleMingledSegments() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new MergeContent());
@@ -941,7 +939,7 @@ public class TestMergeContent {
attributes.put(MergeContent.FRAGMENT_INDEX_ATTRIBUTE, "4");
runner.enqueue("Panama".getBytes("UTF-8"), attributes);
- runner.run(2);
+ runner.run(1);
runner.assertTransferCount(MergeContent.REL_MERGED, 2);
final MockFlowFile assembled = runner.getFlowFilesForRelationship(MergeContent.REL_MERGED).get(0);
@@ -1007,7 +1005,6 @@ public class TestMergeContent {
assembled.assertContentEquals("A Man A Plan A Canal Panama".getBytes("UTF-8"));
}
- @Ignore("This test appears to be a fail...is retuning 1 instead of 2...needs work")
@Test
public void testMergeBasedOnCorrelation() throws IOException, InterruptedException {
final TestRunner runner = TestRunners.newTestRunner(new MergeContent());
@@ -1028,7 +1025,7 @@ public class TestMergeContent {
attributes.put("attr", "b");
runner.enqueue("Panama".getBytes("UTF-8"), attributes);
- runner.run(2);
+ runner.run(1);
runner.assertTransferCount(MergeContent.REL_MERGED, 2);
[15/50] nifi git commit: NIFI-4756: Updated PublishKafkaRecord
processors to include attributes generated from schema write strategy into
the message headers when appropriate
Posted by al...@apache.org.
NIFI-4756: Updated PublishKafkaRecord processors to include attributes generated from schema write strategy into the message headers when appropriate
This closes #2396.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6caf956a
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6caf956a
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6caf956a
Branch: refs/heads/HDF-3.1-maint
Commit: 6caf956a3e6ced23e856b989548b62507a69e69c
Parents: d309819
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Jan 10 09:04:52 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 16:35:20 2018 -0500
----------------------------------------------------------------------
.../WriteAvroSchemaAttributeStrategy.java | 26 ++-
.../processors/kafka/pubsub/PublisherLease.java | 24 +-
.../pubsub/TestPublishKafkaRecord_0_11.java | 2 +-
.../kafka/pubsub/TestPublisherLease.java | 5 +-
.../nifi/processors/kafka/pubsub/TestUtils.java | 45 ----
.../kafka/pubsub/util/MockRecordWriter.java | 2 +-
.../processors/kafka/test/EmbeddedKafka.java | 226 -------------------
.../nifi/processors/kafka/pubsub/TestUtils.java | 45 ----
.../processors/kafka/pubsub/PublisherLease.java | 24 +-
.../pubsub/TestPublishKafkaRecord_1_0.java | 2 +-
.../kafka/pubsub/TestPublisherLease.java | 2 +
.../nifi/processors/kafka/pubsub/TestUtils.java | 45 ----
.../kafka/pubsub/util/MockRecordWriter.java | 2 +-
.../processors/kafka/test/EmbeddedKafka.java | 226 -------------------
.../org/apache/nifi/json/WriteJsonResult.java | 9 +-
15 files changed, 74 insertions(+), 611 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/schema/access/WriteAvroSchemaAttributeStrategy.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/schema/access/WriteAvroSchemaAttributeStrategy.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/schema/access/WriteAvroSchemaAttributeStrategy.java
index d9be673..5f94679 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/schema/access/WriteAvroSchemaAttributeStrategy.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/schema/access/WriteAvroSchemaAttributeStrategy.java
@@ -21,7 +21,9 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.avro.Schema;
@@ -29,6 +31,12 @@ import org.apache.nifi.avro.AvroTypeUtil;
import org.apache.nifi.serialization.record.RecordSchema;
public class WriteAvroSchemaAttributeStrategy implements SchemaAccessWriter {
+ private final Map<RecordSchema, String> avroSchemaTextCache = new LinkedHashMap<RecordSchema, String>() {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<RecordSchema, String> eldest) {
+ return size() > 10;
+ }
+ };
@Override
public void writeHeader(final RecordSchema schema, final OutputStream out) throws IOException {
@@ -36,8 +44,22 @@ public class WriteAvroSchemaAttributeStrategy implements SchemaAccessWriter {
@Override
public Map<String, String> getAttributes(final RecordSchema schema) {
- final Schema avroSchema = AvroTypeUtil.extractAvroSchema(schema);
- final String schemaText = avroSchema.toString();
+ // First, check if schema has the Avro Text available already.
+ final Optional<String> schemaFormat = schema.getSchemaFormat();
+ if (schemaFormat.isPresent() && AvroTypeUtil.AVRO_SCHEMA_FORMAT.equals(schemaFormat.get())) {
+ final Optional<String> schemaText = schema.getSchemaText();
+ if (schemaText.isPresent()) {
+ return Collections.singletonMap("avro.schema", schemaText.get());
+ }
+ }
+
+ String schemaText = avroSchemaTextCache.get(schema);
+ if (schemaText == null) {
+ final Schema avroSchema = AvroTypeUtil.extractAvroSchema(schema);
+ schemaText = avroSchema.toString();
+ avroSchemaTextCache.put(schema, schemaText);
+ }
+
return Collections.singletonMap("avro.schema", schemaText);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
index 72c90d2..2e25129 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -39,6 +40,7 @@ import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
@@ -164,8 +166,10 @@ public class PublisherLease implements Closeable {
recordCount++;
baos.reset();
+ Map<String, String> additionalAttributes = Collections.emptyMap();
try (final RecordSetWriter writer = writerFactory.createWriter(logger, schema, baos)) {
- writer.write(record);
+ final WriteResult writeResult = writer.write(record);
+ additionalAttributes = writeResult.getAttributes();
writer.flush();
}
@@ -173,7 +177,7 @@ public class PublisherLease implements Closeable {
final String key = messageKeyField == null ? null : record.getAsString(messageKeyField);
final byte[] messageKey = (key == null) ? null : key.getBytes(StandardCharsets.UTF_8);
- publish(flowFile, messageKey, messageContent, topic, tracker);
+ publish(flowFile, additionalAttributes, messageKey, messageContent, topic, tracker);
if (tracker.isFailed(flowFile)) {
// If we have a failure, don't try to send anything else.
@@ -195,7 +199,7 @@ public class PublisherLease implements Closeable {
}
}
- private void addHeaders(final FlowFile flowFile, final ProducerRecord<?, ?> record) {
+ private void addHeaders(final FlowFile flowFile, final Map<String, String> additionalAttributes, final ProducerRecord<?, ?> record) {
if (attributeNameRegex == null) {
return;
}
@@ -206,11 +210,23 @@ public class PublisherLease implements Closeable {
headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet));
}
}
+
+ for (final Map.Entry<String, String> entry : additionalAttributes.entrySet()) {
+ if (attributeNameRegex.matcher(entry.getKey()).matches()) {
+ headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet));
+ }
+ }
}
protected void publish(final FlowFile flowFile, final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) {
+ publish(flowFile, Collections.emptyMap(), messageKey, messageContent, topic, tracker);
+ }
+
+ protected void publish(final FlowFile flowFile, final Map<String, String> additionalAttributes,
+ final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) {
+
final ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topic, null, messageKey, messageContent);
- addHeaders(flowFile, record);
+ addHeaders(flowFile, additionalAttributes, record);
producer.send(record, new Callback() {
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_0_11.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_0_11.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_0_11.java
index b7d4abd..9a209d5 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_0_11.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_0_11.java
@@ -177,7 +177,7 @@ public class TestPublishKafkaRecord_0_11 {
runner.assertAllFlowFilesTransferred(PublishKafkaRecord_0_11.REL_SUCCESS, 2);
verify(mockLease, times(2)).publish(any(FlowFile.class), any(RecordSet.class), any(RecordSetWriterFactory.class), any(RecordSchema.class), eq(null), eq(TOPIC_NAME));
- verify(mockLease, times(4)).publish(any(FlowFile.class), eq(null), any(byte[].class), eq(TOPIC_NAME), any(InFlightMessageTracker.class));
+ verify(mockLease, times(4)).publish(any(FlowFile.class), any(Map.class), eq(null), any(byte[].class), eq(TOPIC_NAME), any(InFlightMessageTracker.class));
verify(mockLease, times(1)).complete();
verify(mockLease, times(0)).poison();
verify(mockLease, times(1)).close();
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
index d2b52dd..3ab7abb 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
@@ -38,12 +38,12 @@ import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processors.kafka.pubsub.util.MockRecordParser;
-import org.apache.nifi.processors.kafka.pubsub.util.MockRecordWriter;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.MalformedRecordException;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
@@ -270,13 +270,12 @@ public class TestPublisherLease {
final RecordSet recordSet = reader.createRecordSet();
final RecordSchema schema = reader.getSchema();
- final RecordSetWriterFactory writerService = new MockRecordWriter("person_id, name, age");
-
final String topic = "unit-test";
final String keyField = "person_id";
final RecordSetWriterFactory writerFactory = Mockito.mock(RecordSetWriterFactory.class);
final RecordSetWriter writer = Mockito.mock(RecordSetWriter.class);
+ Mockito.when(writer.write(Mockito.any(Record.class))).thenReturn(WriteResult.of(1, Collections.emptyMap()));
Mockito.when(writerFactory.createWriter(eq(logger), eq(schema), any())).thenReturn(writer);
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
deleted file mode 100644
index 819e3b7..0000000
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.kafka.pubsub;
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
-import sun.misc.Unsafe;
-
-class TestUtils {
-
- public static void setFinalField(Field field, Object instance, Object newValue) throws Exception {
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-
- field.set(instance, newValue);
- }
-
- static Unsafe getUnsafe() {
- try {
- Field f = Unsafe.class.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- return (Unsafe) f.get(null);
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
index 90a909d..0eb8606 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
@@ -108,7 +108,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
@Override
public WriteResult write(Record record) throws IOException {
- return null;
+ return WriteResult.of(1, Collections.emptyMap());
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java
deleted file mode 100644
index a720b11..0000000
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-11-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.kafka.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.Properties;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.zookeeper.server.ServerCnxnFactory;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.server.KafkaConfig;
-import kafka.server.KafkaServerStartable;
-
-/**
- * Embedded Kafka server, primarily to be used for testing.
- */
-public class EmbeddedKafka {
-
- private final KafkaServerStartable kafkaServer;
-
- private final Properties zookeeperConfig;
-
- private final Properties kafkaConfig;
-
- private final ZooKeeperServer zkServer;
-
- private final Logger logger = LoggerFactory.getLogger(EmbeddedKafka.class);
-
- private final int kafkaPort;
-
- private final int zookeeperPort;
-
- private boolean started;
-
- /**
- * Will create instance of the embedded Kafka server. Kafka and Zookeeper
- * configuration properties will be loaded from 'server.properties' and
- * 'zookeeper.properties' located at the root of the classpath.
- */
- public EmbeddedKafka() {
- this(loadPropertiesFromClasspath("/server.properties"), loadPropertiesFromClasspath("/zookeeper.properties"));
- }
-
- /**
- * Will create instance of the embedded Kafka server.
- *
- * @param kafkaConfig
- * Kafka configuration properties
- * @param zookeeperConfig
- * Zookeeper configuration properties
- */
- public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
- this.cleanupKafkaWorkDir();
- this.zookeeperConfig = zookeeperConfig;
- this.kafkaConfig = kafkaConfig;
- this.kafkaPort = this.availablePort();
- this.zookeeperPort = this.availablePort();
-
- this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
- this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
- this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
- this.zkServer = new ZooKeeperServer();
- this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
- }
-
- /**
- *
- * @return port for Kafka server
- */
- public int getKafkaPort() {
- if (!this.started) {
- throw new IllegalStateException("Kafka server is not started. Kafka port can't be determined.");
- }
- return this.kafkaPort;
- }
-
- /**
- *
- * @return port for Zookeeper server
- */
- public int getZookeeperPort() {
- if (!this.started) {
- throw new IllegalStateException("Kafka server is not started. Zookeeper port can't be determined.");
- }
- return this.zookeeperPort;
- }
-
- /**
- * Will start embedded Kafka server. Its data directories will be created
- * at 'kafka-tmp' directory relative to the working directory of the current
- * runtime. The data directories will be deleted upon JVM exit.
- *
- */
- public void start() {
- if (!this.started) {
- logger.info("Starting Zookeeper server");
- this.startZookeeper();
-
- logger.info("Starting Kafka server");
- this.kafkaServer.startup();
-
- logger.info("Embedded Kafka is started at localhost:" + this.kafkaServer.serverConfig().port()
- + ". Zookeeper connection string: " + this.kafkaConfig.getProperty("zookeeper.connect"));
- this.started = true;
- }
- }
-
- /**
- * Will stop embedded Kafka server, cleaning up all working directories.
- */
- public void stop() {
- if (this.started) {
- logger.info("Shutting down Kafka server");
- this.kafkaServer.shutdown();
- this.kafkaServer.awaitShutdown();
- logger.info("Shutting down Zookeeper server");
- this.shutdownZookeeper();
- logger.info("Embedded Kafka is shut down.");
- this.cleanupKafkaWorkDir();
- this.started = false;
- }
- }
-
- /**
- *
- */
- private void cleanupKafkaWorkDir() {
- File kafkaTmp = new File("target/kafka-tmp");
- try {
- FileUtils.deleteDirectory(kafkaTmp);
- } catch (Exception e) {
- logger.warn("Failed to delete " + kafkaTmp.getAbsolutePath());
- }
- }
-
- /**
- * Will start Zookeeper server via {@link ServerCnxnFactory}
- */
- private void startZookeeper() {
- QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
- try {
- quorumConfiguration.parseProperties(this.zookeeperConfig);
-
- ServerConfig configuration = new ServerConfig();
- configuration.readFrom(quorumConfiguration);
-
- FileTxnSnapLog txnLog = new FileTxnSnapLog(new File(configuration.getDataLogDir()), new File(configuration.getDataDir()));
-
- zkServer.setTxnLogFactory(txnLog);
- zkServer.setTickTime(configuration.getTickTime());
- zkServer.setMinSessionTimeout(configuration.getMinSessionTimeout());
- zkServer.setMaxSessionTimeout(configuration.getMaxSessionTimeout());
- ServerCnxnFactory zookeeperConnectionFactory = ServerCnxnFactory.createFactory();
- zookeeperConnectionFactory.configure(configuration.getClientPortAddress(),
- configuration.getMaxClientCnxns());
- zookeeperConnectionFactory.startup(zkServer);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- throw new IllegalStateException("Failed to start Zookeeper server", e);
- }
- }
-
- /**
- * Will shut down Zookeeper server.
- */
- private void shutdownZookeeper() {
- zkServer.shutdown();
- }
-
- /**
- * Will load {@link Properties} from properties file discovered at the
- * provided path relative to the root of the classpath.
- */
- private static Properties loadPropertiesFromClasspath(String path) {
- try {
- Properties kafkaProperties = new Properties();
- kafkaProperties.load(Class.class.getResourceAsStream(path));
- return kafkaProperties;
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
- /**
- * Will determine the available port used by Kafka/Zookeeper servers.
- */
- private int availablePort() {
- ServerSocket s = null;
- try {
- s = new ServerSocket(0);
- s.setReuseAddress(true);
- return s.getLocalPort();
- } catch (Exception e) {
- throw new IllegalStateException("Failed to discover available port.", e);
- } finally {
- try {
- s.close();
- } catch (IOException e) {
- // ignore
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-9-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-9-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-9-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
deleted file mode 100644
index 819e3b7..0000000
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-9-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.kafka.pubsub;
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
-import sun.misc.Unsafe;
-
-class TestUtils {
-
- public static void setFinalField(Field field, Object instance, Object newValue) throws Exception {
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-
- field.set(instance, newValue);
- }
-
- static Unsafe getUnsafe() {
- try {
- Field f = Unsafe.class.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- return (Unsafe) f.get(null);
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
index 2b1cfe2..1c241a4 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublisherLease.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -39,6 +40,7 @@ import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
@@ -163,8 +165,10 @@ public class PublisherLease implements Closeable {
recordCount++;
baos.reset();
+ Map<String, String> additionalAttributes = Collections.emptyMap();
try (final RecordSetWriter writer = writerFactory.createWriter(logger, schema, baos)) {
- writer.write(record);
+ final WriteResult writeResult = writer.write(record);
+ additionalAttributes = writeResult.getAttributes();
writer.flush();
}
@@ -172,7 +176,7 @@ public class PublisherLease implements Closeable {
final String key = messageKeyField == null ? null : record.getAsString(messageKeyField);
final byte[] messageKey = (key == null) ? null : key.getBytes(StandardCharsets.UTF_8);
- publish(flowFile, messageKey, messageContent, topic, tracker);
+ publish(flowFile, additionalAttributes, messageKey, messageContent, topic, tracker);
if (tracker.isFailed(flowFile)) {
// If we have a failure, don't try to send anything else.
@@ -194,7 +198,7 @@ public class PublisherLease implements Closeable {
}
}
- private void addHeaders(final FlowFile flowFile, final ProducerRecord<?, ?> record) {
+ private void addHeaders(final FlowFile flowFile, final Map<String, String> additionalAttributes, final ProducerRecord<?, ?> record) {
if (attributeNameRegex == null) {
return;
}
@@ -205,11 +209,23 @@ public class PublisherLease implements Closeable {
headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet));
}
}
+
+ for (final Map.Entry<String, String> entry : additionalAttributes.entrySet()) {
+ if (attributeNameRegex.matcher(entry.getKey()).matches()) {
+ headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet));
+ }
+ }
}
protected void publish(final FlowFile flowFile, final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) {
+ publish(flowFile, Collections.emptyMap(), messageKey, messageContent, topic, tracker);
+ }
+
+ protected void publish(final FlowFile flowFile, final Map<String, String> additionalAttributes,
+ final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) {
+
final ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topic, null, messageKey, messageContent);
- addHeaders(flowFile, record);
+ addHeaders(flowFile, additionalAttributes, record);
producer.send(record, new Callback() {
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_1_0.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_1_0.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_1_0.java
index 45439cc..abadc89 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_1_0.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublishKafkaRecord_1_0.java
@@ -177,7 +177,7 @@ public class TestPublishKafkaRecord_1_0 {
runner.assertAllFlowFilesTransferred(PublishKafkaRecord_1_0.REL_SUCCESS, 2);
verify(mockLease, times(2)).publish(any(FlowFile.class), any(RecordSet.class), any(RecordSetWriterFactory.class), any(RecordSchema.class), eq(null), eq(TOPIC_NAME));
- verify(mockLease, times(4)).publish(any(FlowFile.class), eq(null), any(byte[].class), eq(TOPIC_NAME), any(InFlightMessageTracker.class));
+ verify(mockLease, times(4)).publish(any(FlowFile.class), any(Map.class), eq(null), any(byte[].class), eq(TOPIC_NAME), any(InFlightMessageTracker.class));
verify(mockLease, times(1)).complete();
verify(mockLease, times(0)).poison();
verify(mockLease, times(1)).close();
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
index b2e1b0e..2fbf539 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestPublisherLease.java
@@ -43,6 +43,7 @@ import org.apache.nifi.serialization.MalformedRecordException;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
@@ -274,6 +275,7 @@ public class TestPublisherLease {
final RecordSetWriterFactory writerFactory = Mockito.mock(RecordSetWriterFactory.class);
final RecordSetWriter writer = Mockito.mock(RecordSetWriter.class);
+ Mockito.when(writer.write(Mockito.any(Record.class))).thenReturn(WriteResult.of(1, Collections.emptyMap()));
Mockito.when(writerFactory.createWriter(eq(logger), eq(schema), any())).thenReturn(writer);
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
deleted file mode 100644
index 819e3b7..0000000
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestUtils.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.kafka.pubsub;
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
-import sun.misc.Unsafe;
-
-class TestUtils {
-
- public static void setFinalField(Field field, Object instance, Object newValue) throws Exception {
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-
- field.set(instance, newValue);
- }
-
- static Unsafe getUnsafe() {
- try {
- Field f = Unsafe.class.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- return (Unsafe) f.get(null);
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
index 90a909d..0eb8606 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
@@ -108,7 +108,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
@Override
public WriteResult write(Record record) throws IOException {
- return null;
+ return WriteResult.of(1, Collections.emptyMap());
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java
deleted file mode 100644
index a720b11..0000000
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-1-0-processors/src/test/java/org/apache/nifi/processors/kafka/test/EmbeddedKafka.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.kafka.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.Properties;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.zookeeper.server.ServerCnxnFactory;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.server.KafkaConfig;
-import kafka.server.KafkaServerStartable;
-
-/**
- * Embedded Kafka server, primarily to be used for testing.
- */
-public class EmbeddedKafka {
-
- private final KafkaServerStartable kafkaServer;
-
- private final Properties zookeeperConfig;
-
- private final Properties kafkaConfig;
-
- private final ZooKeeperServer zkServer;
-
- private final Logger logger = LoggerFactory.getLogger(EmbeddedKafka.class);
-
- private final int kafkaPort;
-
- private final int zookeeperPort;
-
- private boolean started;
-
- /**
- * Will create instance of the embedded Kafka server. Kafka and Zookeeper
- * configuration properties will be loaded from 'server.properties' and
- * 'zookeeper.properties' located at the root of the classpath.
- */
- public EmbeddedKafka() {
- this(loadPropertiesFromClasspath("/server.properties"), loadPropertiesFromClasspath("/zookeeper.properties"));
- }
-
- /**
- * Will create instance of the embedded Kafka server.
- *
- * @param kafkaConfig
- * Kafka configuration properties
- * @param zookeeperConfig
- * Zookeeper configuration properties
- */
- public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
- this.cleanupKafkaWorkDir();
- this.zookeeperConfig = zookeeperConfig;
- this.kafkaConfig = kafkaConfig;
- this.kafkaPort = this.availablePort();
- this.zookeeperPort = this.availablePort();
-
- this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
- this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
- this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
- this.zkServer = new ZooKeeperServer();
- this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
- }
-
- /**
- *
- * @return port for Kafka server
- */
- public int getKafkaPort() {
- if (!this.started) {
- throw new IllegalStateException("Kafka server is not started. Kafka port can't be determined.");
- }
- return this.kafkaPort;
- }
-
- /**
- *
- * @return port for Zookeeper server
- */
- public int getZookeeperPort() {
- if (!this.started) {
- throw new IllegalStateException("Kafka server is not started. Zookeeper port can't be determined.");
- }
- return this.zookeeperPort;
- }
-
- /**
- * Will start embedded Kafka server. Its data directories will be created
- * at 'kafka-tmp' directory relative to the working directory of the current
- * runtime. The data directories will be deleted upon JVM exit.
- *
- */
- public void start() {
- if (!this.started) {
- logger.info("Starting Zookeeper server");
- this.startZookeeper();
-
- logger.info("Starting Kafka server");
- this.kafkaServer.startup();
-
- logger.info("Embedded Kafka is started at localhost:" + this.kafkaServer.serverConfig().port()
- + ". Zookeeper connection string: " + this.kafkaConfig.getProperty("zookeeper.connect"));
- this.started = true;
- }
- }
-
- /**
- * Will stop embedded Kafka server, cleaning up all working directories.
- */
- public void stop() {
- if (this.started) {
- logger.info("Shutting down Kafka server");
- this.kafkaServer.shutdown();
- this.kafkaServer.awaitShutdown();
- logger.info("Shutting down Zookeeper server");
- this.shutdownZookeeper();
- logger.info("Embedded Kafka is shut down.");
- this.cleanupKafkaWorkDir();
- this.started = false;
- }
- }
-
- /**
- *
- */
- private void cleanupKafkaWorkDir() {
- File kafkaTmp = new File("target/kafka-tmp");
- try {
- FileUtils.deleteDirectory(kafkaTmp);
- } catch (Exception e) {
- logger.warn("Failed to delete " + kafkaTmp.getAbsolutePath());
- }
- }
-
- /**
- * Will start Zookeeper server via {@link ServerCnxnFactory}
- */
- private void startZookeeper() {
- QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
- try {
- quorumConfiguration.parseProperties(this.zookeeperConfig);
-
- ServerConfig configuration = new ServerConfig();
- configuration.readFrom(quorumConfiguration);
-
- FileTxnSnapLog txnLog = new FileTxnSnapLog(new File(configuration.getDataLogDir()), new File(configuration.getDataDir()));
-
- zkServer.setTxnLogFactory(txnLog);
- zkServer.setTickTime(configuration.getTickTime());
- zkServer.setMinSessionTimeout(configuration.getMinSessionTimeout());
- zkServer.setMaxSessionTimeout(configuration.getMaxSessionTimeout());
- ServerCnxnFactory zookeeperConnectionFactory = ServerCnxnFactory.createFactory();
- zookeeperConnectionFactory.configure(configuration.getClientPortAddress(),
- configuration.getMaxClientCnxns());
- zookeeperConnectionFactory.startup(zkServer);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- throw new IllegalStateException("Failed to start Zookeeper server", e);
- }
- }
-
- /**
- * Will shut down Zookeeper server.
- */
- private void shutdownZookeeper() {
- zkServer.shutdown();
- }
-
- /**
- * Will load {@link Properties} from properties file discovered at the
- * provided path relative to the root of the classpath.
- */
- private static Properties loadPropertiesFromClasspath(String path) {
- try {
- Properties kafkaProperties = new Properties();
- kafkaProperties.load(Class.class.getResourceAsStream(path));
- return kafkaProperties;
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
- /**
- * Will determine the available port used by Kafka/Zookeeper servers.
- */
- private int availablePort() {
- ServerSocket s = null;
- try {
- s = new ServerSocket(0);
- s.setReuseAddress(true);
- return s.getLocalPort();
- } catch (Exception e) {
- throw new IllegalStateException("Failed to discover available port.", e);
- } finally {
- try {
- s.close();
- } catch (IOException e) {
- // ignore
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6caf956a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
index fc84181..41a72c7 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.math.BigInteger;
import java.text.DateFormat;
-import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -118,30 +117,26 @@ public class WriteJsonResult extends AbstractRecordSetWriter implements RecordSe
public Map<String, String> writeRecord(final Record record) throws IOException {
// If we are not writing an active record set, then we need to ensure that we write the
// schema information.
- boolean firstRecord = false;
if (!isActiveRecordSet()) {
generator.flush();
schemaAccess.writeHeader(recordSchema, getOutputStream());
- firstRecord = true;
}
writeRecord(record, recordSchema, generator, g -> g.writeStartObject(), g -> g.writeEndObject(), true);
- return firstRecord ? schemaAccess.getAttributes(recordSchema) : Collections.emptyMap();
+ return schemaAccess.getAttributes(recordSchema);
}
@Override
public WriteResult writeRawRecord(final Record record) throws IOException {
// If we are not writing an active record set, then we need to ensure that we write the
// schema information.
- boolean firstRecord = false;
if (!isActiveRecordSet()) {
generator.flush();
schemaAccess.writeHeader(recordSchema, getOutputStream());
- firstRecord = true;
}
writeRecord(record, recordSchema, generator, g -> g.writeStartObject(), g -> g.writeEndObject(), false);
- final Map<String, String> attributes = firstRecord ? schemaAccess.getAttributes(recordSchema) : Collections.emptyMap();
+ final Map<String, String> attributes = schemaAccess.getAttributes(recordSchema);
return WriteResult.of(incrementRecordCount(), attributes);
}
[43/50] nifi git commit: NIFI-4959: Remove flowfiles and close
connection for Bad Requests causing IOException
Posted by al...@apache.org.
NIFI-4959: Remove flowfiles and close connection for Bad Requests causing IOException
This closes #2544.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/8a97e3eb
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/8a97e3eb
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/8a97e3eb
Branch: refs/heads/HDF-3.1-maint
Commit: 8a97e3eb774755c07d1f40b474f2f8c800fdf362
Parents: 5d53b45
Author: Marco Gaido <ma...@gmail.com>
Authored: Wed Mar 14 16:27:17 2018 +0100
Committer: Matt Gilman <ma...@gmail.com>
Committed: Thu Apr 5 13:37:37 2018 -0400
----------------------------------------------------------------------
.../processors/standard/HandleHttpRequest.java | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/8a97e3eb/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index 5a09dab..ad1b171 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -39,6 +39,7 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.standard.util.HTTPUtils;
import org.apache.nifi.ssl.RestrictedSSLContextService;
import org.apache.nifi.ssl.SSLContextService;
+import org.apache.nifi.stream.io.StreamUtils;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -58,6 +59,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -513,12 +515,26 @@ public class HandleHttpRequest extends AbstractProcessor {
final long start = System.nanoTime();
final HttpServletRequest request = container.getRequest();
FlowFile flowFile = session.create();
- try {
- flowFile = session.importFrom(request.getInputStream(), flowFile);
+ try (OutputStream flowFileOut = session.write(flowFile)) {
+ StreamUtils.copy(request.getInputStream(), flowFileOut);
} catch (final IOException e) {
+ // There may be many reasons which can produce an IOException on the HTTP stream and in some of them, eg.
+ // bad requests, the connection to the client is not closed. In order to address also these cases, we try
+ // and answer with a BAD_REQUEST, which lets the client know that the request has not been correctly
+ // processed and makes it aware that the connection can be closed.
getLogger().error("Failed to receive content from HTTP Request from {} due to {}",
new Object[]{request.getRemoteAddr(), e});
session.remove(flowFile);
+
+ try {
+ HttpServletResponse response = container.getResponse();
+ response.sendError(Status.BAD_REQUEST.getStatusCode());
+ response.flushBuffer();
+ container.getContext().complete();
+ } catch (final IOException ioe) {
+ getLogger().warn("Failed to send HTTP response to {} due to {}",
+ new Object[]{request.getRemoteAddr(), ioe});
+ }
return;
}
[11/50] nifi git commit: NIFI-4769: Use FlowFile for EL at Fetch and
PutAzureBlobStorage
Posted by al...@apache.org.
NIFI-4769: Use FlowFile for EL at Fetch and PutAzureBlobStorage
This commit add back the existing capability for those Processors to use
incoming FlowFile attributes to compute account name and account key,
which had been removed by NIFI-4004.
Also, the same capability is added for SAS token.
This closes #2400.
Signed-off-by: Koji Kawamura <ij...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/e77e66e8
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/e77e66e8
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/e77e66e8
Branch: refs/heads/HDF-3.1-maint
Commit: e77e66e8eb3ab10061bd3691e754715c480634f4
Parents: 6847c9e
Author: Koji Kawamura <ij...@apache.org>
Authored: Fri Jan 12 11:03:05 2018 +0900
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 15:01:01 2018 -0500
----------------------------------------------------------------------
.../azure/storage/FetchAzureBlobStorage.java | 2 +-
.../azure/storage/ListAzureBlobStorage.java | 2 +-
.../azure/storage/PutAzureBlobStorage.java | 2 +-
.../azure/storage/utils/AzureStorageUtils.java | 33 ++++++++++++++++----
4 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e77e66e8/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/FetchAzureBlobStorage.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/FetchAzureBlobStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/FetchAzureBlobStorage.java
index d960f7d..89d22f3 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/FetchAzureBlobStorage.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/FetchAzureBlobStorage.java
@@ -66,7 +66,7 @@ public class FetchAzureBlobStorage extends AbstractAzureBlobProcessor {
AtomicReference<Exception> storedException = new AtomicReference<>();
try {
- CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger());
+ CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
CloudBlobContainer container = blobClient.getContainerReference(containerName);
final Map<String, String> attributes = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/nifi/blob/e77e66e8/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/ListAzureBlobStorage.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/ListAzureBlobStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/ListAzureBlobStorage.java
index 41f347c..2cb6316 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/ListAzureBlobStorage.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/ListAzureBlobStorage.java
@@ -151,7 +151,7 @@ public class ListAzureBlobStorage extends AbstractListProcessor<BlobInfo> {
}
final List<BlobInfo> listing = new ArrayList<>();
try {
- CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger());
+ CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null);
CloudBlobContainer container = blobClient.getContainerReference(containerName);
for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, null)) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/e77e66e8/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureBlobStorage.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureBlobStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureBlobStorage.java
index a316256..c41733b 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureBlobStorage.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureBlobStorage.java
@@ -72,7 +72,7 @@ public class PutAzureBlobStorage extends AbstractAzureBlobProcessor {
AtomicReference<Exception> storedException = new AtomicReference<>();
try {
- CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger());
+ CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
CloudBlobContainer container = blobClient.getContainerReference(containerName);
CloudBlob blob = container.getBlockBlobReference(blobPath);
http://git-wip-us.apache.org/repos/asf/nifi/blob/e77e66e8/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
index 360afbf..95b6d7e 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.util.StandardValidators;
@@ -42,7 +43,7 @@ public final class AzureStorageUtils {
public static final PropertyDescriptor ACCOUNT_KEY = new PropertyDescriptor.Builder().name("storage-account-key").displayName("Storage Account Key")
.description("The storage account key. This is an admin-like password providing access to every container in this account. It is recommended " +
"one uses Shared Access Signature (SAS) token instead for fine-grained control with policies. " +
- "There are certain risks in allowing the account key to be stored as a flowfile" +
+ "There are certain risks in allowing the account key to be stored as a flowfile " +
"attribute. While it does provide for a more flexible flow by allowing the account key to " +
"be fetched dynamically from a flow file attribute, care must be taken to restrict access to " +
"the event provenance data (e.g. by strictly controlling the policies governing provenance for this Processor). " +
@@ -63,7 +64,12 @@ public final class AzureStorageUtils {
public static final PropertyDescriptor PROP_SAS_TOKEN = new PropertyDescriptor.Builder()
.name("storage-sas-token")
.displayName("SAS Token")
- .description("Shared Access Signature token, including the leading '?'. Specify either SAS Token (recommended) or Account Key")
+ .description("Shared Access Signature token, including the leading '?'. Specify either SAS Token (recommended) or Account Key. " +
+ "There are certain risks in allowing the SAS token to be stored as a flowfile " +
+ "attribute. While it does provide for a more flexible flow by allowing the account name to " +
+ "be fetched dynamically from a flowfile attribute, care must be taken to restrict access to " +
+ "the event provenance data (e.g. by strictly controlling the policies governing provenance for this Processor). " +
+ "In addition, the provenance repositories may be put on encrypted disk partitions.")
.required(false)
.expressionLanguageSupported(true)
.sensitive(true)
@@ -78,10 +84,25 @@ public final class AzureStorageUtils {
// do not instantiate
}
- public static CloudBlobClient createCloudBlobClient(ProcessContext context, ComponentLog logger) {
- final String accountName = context.getProperty(AzureStorageUtils.ACCOUNT_NAME).evaluateAttributeExpressions().getValue();
- final String accountKey = context.getProperty(AzureStorageUtils.ACCOUNT_KEY).evaluateAttributeExpressions().getValue();
- final String sasToken = context.getProperty(AzureStorageUtils.PROP_SAS_TOKEN).evaluateAttributeExpressions().getValue();
+ /**
+ * Create CloudBlobClient instance.
+ * @param flowFile An incoming FlowFile can be used for NiFi Expression Language evaluation to derive
+ * Account Name, Account Key or SAS Token. This can be null if not available.
+ */
+ public static CloudBlobClient createCloudBlobClient(ProcessContext context, ComponentLog logger, FlowFile flowFile) {
+ final String accountName;
+ final String accountKey;
+ final String sasToken;
+
+ if (flowFile == null) {
+ accountName = context.getProperty(AzureStorageUtils.ACCOUNT_NAME).evaluateAttributeExpressions().getValue();
+ accountKey = context.getProperty(AzureStorageUtils.ACCOUNT_KEY).evaluateAttributeExpressions().getValue();
+ sasToken = context.getProperty(AzureStorageUtils.PROP_SAS_TOKEN).evaluateAttributeExpressions().getValue();
+ } else {
+ accountName = context.getProperty(AzureStorageUtils.ACCOUNT_NAME).evaluateAttributeExpressions(flowFile).getValue();
+ accountKey = context.getProperty(AzureStorageUtils.ACCOUNT_KEY).evaluateAttributeExpressions(flowFile).getValue();
+ sasToken = context.getProperty(AzureStorageUtils.PROP_SAS_TOKEN).evaluateAttributeExpressions(flowFile).getValue();
+ }
CloudBlobClient cloudBlobClient;
[16/50] nifi git commit: NIFI-4761 Added HostHeaderHandler regression
unit tests before adding new functionality. Added logic for parsing
nifi.web.proxy.host property. Added default property in nifi.properties and
pom.xml. Added logic for IPv6 and custom
Posted by al...@apache.org.
NIFI-4761
Added HostHeaderHandler regression unit tests before adding new functionality.
Added logic for parsing nifi.web.proxy.host property.
Added default property in nifi.properties and pom.xml.
Added logic for IPv6 and custom default hostnames.
Improved error messaging.
Added HostHeaderHandler unit tests.
Disabled HostHeaderSanitizationCustomizer in HTTP mode.
Fixed HTML escaping in error message.
Improved error message.
Added failing unit test for parsing custom hostnames.
Fixed custom hostname parsing.
Fixed unit tests.
Added TODO for IPv6 custom hostname parsing and unit test.
Added IPv6 custom hostname parsing and unit tests.
Fixed checkstyle issues.
Removed empty element in host list when no value defined.
Improved error message formatting.
Added unit tests.
Removed HostHeaderSanitizationCustomizer.
Removed InvalidPropertiesFormatException from NiFiProperties.
Removed InvalidPropertiesFormatException from HostHeaderHandler.
This closes #2415
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/bf854bab
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/bf854bab
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/bf854bab
Branch: refs/heads/HDF-3.1-maint
Commit: bf854bab8de9315164492b2bb4a72a9cd1167053
Parents: 6caf956
Author: Andy LoPresto <al...@apache.org>
Authored: Wed Jan 17 11:36:00 2018 -0800
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Jan 19 11:06:21 2018 -0500
----------------------------------------------------------------------
.../org/apache/nifi/util/NiFiProperties.java | 55 +++-
.../StandardNiFiPropertiesGroovyTest.groovy | 130 +++++++++
.../nifi-framework/nifi-resources/pom.xml | 1 +
.../src/main/resources/conf/nifi.properties | 1 +
.../nifi/web/server/HostHeaderHandler.java | 286 ++++++++++++++++--
.../HostHeaderSanitizationCustomizer.java | 87 ------
.../org/apache/nifi/web/server/JettyServer.java | 43 +--
.../web/server/HostHeaderHandlerTest.groovy | 287 +++++++++++++++++++
8 files changed, 744 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index 55cae14..46704ef 100644
--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -178,6 +178,7 @@ public abstract class NiFiProperties {
public static final String WEB_THREADS = "nifi.web.jetty.threads";
public static final String WEB_MAX_HEADER_SIZE = "nifi.web.max.header.size";
public static final String WEB_PROXY_CONTEXT_PATH = "nifi.web.proxy.context.path";
+ public static final String WEB_PROXY_HOST = "nifi.web.proxy.host";
// ui properties
public static final String UI_BANNER_TEXT = "nifi.ui.banner.text";
@@ -287,6 +288,7 @@ public abstract class NiFiProperties {
// Kerberos defaults
public static final String DEFAULT_KERBEROS_AUTHENTICATION_EXPIRATION = "12 hours";
+
/**
* Retrieves the property value for the given property key.
*
@@ -587,6 +589,25 @@ public abstract class NiFiProperties {
return sslPort;
}
+ public boolean isHTTPSConfigured() {
+ return getSslPort() != null;
+ }
+
+ /**
+ * Determines the HTTP/HTTPS port NiFi is configured to bind to. Prefers the HTTPS port. Throws an exception if neither is configured.
+ *
+ * @return the configured port number
+ */
+ public Integer getConfiguredHttpOrHttpsPort() throws RuntimeException {
+ if (getSslPort() != null) {
+ return getSslPort();
+ } else if (getPort() != null) {
+ return getPort();
+ } else {
+ throw new RuntimeException("The HTTP or HTTPS port must be configured");
+ }
+ }
+
public String getWebMaxHeaderSize() {
return getProperty(WEB_MAX_HEADER_SIZE, DEFAULT_WEB_MAX_HEADER_SIZE);
}
@@ -1287,8 +1308,40 @@ public abstract class NiFiProperties {
}
/**
- * Returns the whitelisted proxy context paths as a comma-delimited string. The paths have been normalized to the form {@code /some/context/path}.
+ * Returns the whitelisted proxy hostnames (and IP addresses) as a comma-delimited string.
+ * The hosts have been normalized to the form {@code somehost.com}, {@code somehost.com:port}, or {@code 127.0.0.1}.
+ * <p>
+ * Note: Calling {@code NiFiProperties.getProperty(NiFiProperties.WEB_PROXY_HOST)} will not normalize the hosts.
+ *
+ * @return the hostname(s)
+ */
+ public String getWhitelistedHosts() {
+ return StringUtils.join(getWhitelistedHostsAsList(), ",");
+ }
+
+ /**
+ * Returns the whitelisted proxy hostnames (and IP addresses) as a List. The hosts have been normalized to the form {@code somehost.com}, {@code somehost.com:port}, or {@code 127.0.0.1}.
*
+ * @return the hostname(s)
+ */
+ public List<String> getWhitelistedHostsAsList() {
+ String rawProperty = getProperty(WEB_PROXY_HOST, "");
+ List<String> hosts = Arrays.asList(rawProperty.split(","));
+ return hosts.stream()
+ .map(this::normalizeHost).filter(host -> !StringUtils.isBlank(host)).collect(Collectors.toList());
+ }
+
+ String normalizeHost(String host) {
+ if (host == null || host.equalsIgnoreCase("")) {
+ return "";
+ } else {
+ return host.trim();
+ }
+ }
+
+ /**
+ * Returns the whitelisted proxy context paths as a comma-delimited string. The paths have been normalized to the form {@code /some/context/path}.
+ * <p>
* Note: Calling {@code NiFiProperties.getProperty(NiFiProperties.WEB_PROXY_CONTEXT_PATH)} will not normalize the paths.
*
* @return the path(s)
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
index 64c4f0f..478dc2a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
@@ -433,4 +433,134 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
// Assert
assert normalizedContextPath == empty
}
+
+ @Test
+ void testShouldNormalizeProxyHostProperty() {
+ // Arrange
+ String extraSpaceHostname = "somehost.com "
+ Properties rawProps = new Properties(["nifi.web.proxy.host": extraSpaceHostname])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${extraSpaceHostname}]")
+
+ // Act
+ String normalizedHostname = props.getWhitelistedHosts()
+ logger.info("Read from NiFiProperties instance: ${normalizedHostname}")
+
+ // Assert
+ assert extraSpaceHostname.startsWith(normalizedHostname)
+ assert extraSpaceHostname.length() == normalizedHostname.length() + 2
+ }
+
+ @Test
+ void testShouldHandleNormalizedProxyHostProperty() {
+ // Arrange
+ String hostname = "somehost.com"
+ Properties rawProps = new Properties(["nifi.web.proxy.host": hostname])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${hostname}]")
+
+ // Act
+ String normalizedHostname = props.getWhitelistedHosts()
+ logger.info("Read from NiFiProperties instance: ${normalizedHostname}")
+
+ // Assert
+ assert hostname == normalizedHostname
+ }
+
+ @Test
+ void testShouldNormalizeMultipleProxyHostsInProperty() {
+ // Arrange
+ String extraSpaceHostname = "somehost.com "
+ String normalHostname = "someotherhost.com"
+ String hostnameWithPort = "otherhost.com:1234"
+ String extraSpaceHostnameWithPort = " anotherhost.com:9999"
+ List<String> hosts = [extraSpaceHostname, normalHostname, hostnameWithPort, extraSpaceHostnameWithPort]
+ String combinedHosts = hosts.join(",")
+ Properties rawProps = new Properties(["nifi.web.proxy.host": combinedHosts])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${combinedHosts}]")
+
+ // Act
+ String normalizedHostname = props.getWhitelistedHosts()
+ logger.info("Read from NiFiProperties instance: ${normalizedHostname}")
+
+ // Assert
+ def splitHosts = normalizedHostname.split(",")
+ def expectedValues = hosts*.trim()
+ splitHosts.every {
+ assert it.trim() == it
+ assert expectedValues.contains(it)
+ }
+ }
+
+ @Test
+ void testShouldHandleNormalizedProxyHostPropertyAsList() {
+ // Arrange
+ String normalHostname = "someotherhost.com"
+ Properties rawProps = new Properties(["nifi.web.proxy.host": normalHostname])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${normalHostname}]")
+
+ // Act
+ def listOfHosts = props.getWhitelistedHostsAsList()
+ logger.info("Read from NiFiProperties instance: ${listOfHosts}")
+
+ // Assert
+ assert listOfHosts.size() == 1
+ assert listOfHosts.contains(normalHostname)
+ }
+
+ @Test
+ void testShouldNormalizeMultipleProxyHostsInPropertyAsList() {
+ // Arrange
+ String extraSpaceHostname = "somehost.com "
+ String normalHostname = "someotherhost.com"
+ String hostnameWithPort = "otherhost.com:1234"
+ String extraSpaceHostnameWithPort = " anotherhost.com:9999"
+ List<String> hosts = [extraSpaceHostname, normalHostname, hostnameWithPort, extraSpaceHostnameWithPort]
+ String combinedHosts = hosts.join(",")
+ Properties rawProps = new Properties(["nifi.web.proxy.host": combinedHosts])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${combinedHosts}]")
+
+ // Act
+ def listOfHosts = props.getWhitelistedHostsAsList()
+ logger.info("Read from NiFiProperties instance: ${listOfHosts}")
+
+ // Assert
+ assert listOfHosts.size() == 4
+ assert listOfHosts.containsAll([extraSpaceHostname[0..-3], normalHostname, hostnameWithPort, extraSpaceHostnameWithPort[2..-1]])
+ }
+
+ @Test
+ void testShouldHandleNormalizingEmptyProxyHostProperty() {
+ // Arrange
+ String empty = ""
+ Properties rawProps = new Properties(["nifi.web.proxy.host": empty])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${empty}]")
+
+ // Act
+ String normalizedHost = props.getWhitelistedHosts()
+ logger.info("Read from NiFiProperties instance: ${normalizedHost}")
+
+ // Assert
+ assert normalizedHost == empty
+ }
+
+ @Test
+ void testShouldReturnEmptyProxyHostPropertyAsList() {
+ // Arrange
+ String empty = ""
+ Properties rawProps = new Properties(["nifi.web.proxy.host": empty])
+ NiFiProperties props = new StandardNiFiProperties(rawProps)
+ logger.info("Created a NiFiProperties instance with raw proxy host property [${empty}]")
+
+ // Act
+ def hosts = props.getWhitelistedHostsAsList()
+ logger.info("Read from NiFiProperties instance: ${hosts}")
+
+ // Assert
+ assert hosts.size() == 0
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
index 959f552..0d03fcd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
@@ -133,6 +133,7 @@
<nifi.web.jetty.threads>200</nifi.web.jetty.threads>
<nifi.web.max.header.size>16 KB</nifi.web.max.header.size>
<nifi.web.proxy.context.path />
+ <nifi.web.proxy.host />
<!-- nifi.properties: security properties -->
<nifi.security.keystore />
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
index f6a1ed5..868ec0a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
@@ -140,6 +140,7 @@ nifi.web.jetty.working.directory=${nifi.jetty.work.dir}
nifi.web.jetty.threads=${nifi.web.jetty.threads}
nifi.web.max.header.size=${nifi.web.max.header.size}
nifi.web.proxy.context.path=${nifi.web.proxy.context.path}
+nifi.web.proxy.host=${nifi.web.proxy.host}
# security properties #
nifi.sensitive.props.key=
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
index 61e825d..9a07fcb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
@@ -16,20 +16,31 @@
*/
package org.apache.nifi.web.server;
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.conn.util.InetAddressUtils;
+import org.apache.nifi.util.NiFiProperties;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.ScopedHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang3.StringEscapeUtils;
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.handler.ScopedHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.stream.Collectors;
public class HostHeaderHandler extends ScopedHandler {
private static final Logger logger = LoggerFactory.getLogger(HostHeaderHandler.class);
@@ -39,38 +50,149 @@ public class HostHeaderHandler extends ScopedHandler {
private final List<String> validHosts;
/**
+ * Instantiates a handler with a given server name and port 0.
+ *
* @param serverName the {@code serverName} to set on the request (the {@code serverPort} will not be set)
+ * @deprecated Use {@link #HostHeaderHandler(NiFiProperties)} which accepts a {@link NiFiProperties} object to allow for custom network interface binding.
*/
public HostHeaderHandler(String serverName) {
this(serverName, 0);
}
/**
+ * Instantiates a handler with a given server name and port.
+ *
* @param serverName the {@code serverName} to set on the request
* @param serverPort the {@code serverPort} to set on the request
+ * @deprecated Use {@link #HostHeaderHandler(NiFiProperties)} which accepts a {@link NiFiProperties} object to allow for custom network interface binding.
*/
public HostHeaderHandler(String serverName, int serverPort) {
this.serverName = Objects.requireNonNull(serverName);
this.serverPort = serverPort;
- validHosts = new ArrayList<>();
+ // Default values across generic instances
+ validHosts = generateDefaultHostnames(null);
+
+ // specified host:port
validHosts.add(serverName.toLowerCase());
validHosts.add(serverName.toLowerCase() + ":" + serverPort);
- // Sometimes the hostname is left empty but the port is always populated
- validHosts.add("localhost");
- validHosts.add("localhost:" + serverPort);
- // Different from customizer -- empty is ok here
+
+ // empty is ok here
validHosts.add("");
- try {
- validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase());
- validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase() + ":" + serverPort);
- } catch (final Exception e) {
- logger.warn("Failed to determine local hostname.", e);
- }
logger.info("Created " + this.toString());
}
+ /**
+ * Instantiates a handler which accepts incoming requests with a host header that is empty or contains one of the
+ * valid hosts. See the Apache NiFi Admin Guide for instructions on how to set valid hostnames and IP addresses.
+ *
+ * @param niFiProperties the NiFiProperties
+ */
+ public HostHeaderHandler(NiFiProperties niFiProperties) {
+ this.serverName = Objects.requireNonNull(determineServerHostname(niFiProperties));
+ this.serverPort = determineServerPort(niFiProperties);
+
+ // Default values across generic instances
+ List<String> hosts = generateDefaultHostnames(niFiProperties);
+
+ // The value from nifi.web.http|https.host
+ hosts.add(serverName.toLowerCase());
+ hosts.add(serverName.toLowerCase() + ":" + serverPort);
+
+ // The value(s) from nifi.web.proxy.host
+ hosts.addAll(parseCustomHostnames(niFiProperties));
+
+ // empty is ok here
+ hosts.add("");
+
+ this.validHosts = uniqueList(hosts);
+ logger.info("Determined {} valid hostnames and IP addresses for incoming headers: {}", new Object[]{validHosts.size(), StringUtils.join(validHosts, ", ")});
+
+ logger.debug("Created " + this.toString());
+ }
+
+ /**
+ * Returns the list of parsed custom hostnames from {@code nifi.web.proxy.host} in {@link NiFiProperties}.
+ * This list is deduplicated (if a host {@code somehost.com:1234} is provided, it will show twice, as the "portless"
+ * version {@code somehost.com} is also generated). IPv6 addresses are only modified if they adhere to the strict
+ * formatting using {@code []} around the address as specified in RFC 5952 Section 6 (i.e.
+ * {@code [1234.5678.90AB.CDEF.1234.5678.90AB.CDEF]:1234} will insert
+ * {@code [1234.5678.90AB.CDEF.1234.5678.90AB.CDEF]} as well).
+ *
+ * @param niFiProperties the properties object
+ * @return the list of parsed custom hostnames
+ */
+ List<String> parseCustomHostnames(NiFiProperties niFiProperties) {
+ // Load the custom hostnames from the properties
+ List<String> customHostnames = niFiProperties.getWhitelistedHostsAsList();
+
+ /* Each IPv4 address and hostname may have the port associated, so duplicate the list and trim the port
+ * (the port may be different from the port NiFi is running on if provided by a proxy, etc.) IPv6 addresses
+ * are not modified.
+ */
+ List<String> portlessHostnames = customHostnames.stream().map(hostname -> {
+ if (isIPv6Address(hostname)) {
+ return hostname;
+ } else {
+ return StringUtils.substringBeforeLast(hostname, ":");
+ }
+ }
+ ).collect(Collectors.toList());
+
+ customHostnames.addAll(portlessHostnames);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Parsed {} custom hostnames from nifi.web.proxy.host: {}", new Object[]{customHostnames.size(), StringUtils.join(customHostnames, ", ")});
+ }
+ return uniqueList(customHostnames);
+ }
+
+ /**
+ * Returns a unique {@code List} of the elements maintaining the original order.
+ *
+ * @param duplicateList a list that may contain duplicate elements
+ * @return a list maintaining the original order which no longer contains duplicate elements
+ */
+ private static List<String> uniqueList(List<String> duplicateList) {
+ return new ArrayList<>(new LinkedHashSet<>(duplicateList));
+ }
+
+ /**
+ * Returns true if the provided address is an IPv6 address (or could be interpreted as one). This method is more
+ * lenient than {@link InetAddressUtils#isIPv6Address(String)} because of different interpretations of IPv4-mapped
+ * IPv6 addresses.
+ *
+ * See RFC 5952 Section 4 for more information on textual representation of the IPv6 addresses.
+ *
+ * @param address the address in text form
+ * @return true if the address is or could be parsed as an IPv6 address
+ */
+ static boolean isIPv6Address(String address) {
+ // Note: InetAddressUtils#isIPv4MappedIPv64Address() fails on addresses that do not compress the leading 0:0:0... to ::
+ // Expanded for debugging purposes
+ boolean isNormalIPv6 = InetAddressUtils.isIPv6Address(address);
+
+ // If the last two hextets are written in IPv4 form, treat it as an IPv6 address as well
+ String everythingAfterLastColon = StringUtils.substringAfterLast(address, ":");
+ boolean isIPv4 = InetAddressUtils.isIPv4Address(everythingAfterLastColon);
+ boolean isIPv4Mapped = InetAddressUtils.isIPv4MappedIPv64Address(everythingAfterLastColon);
+ boolean isCompressable = address.contains("0:0") && !address.contains("::");
+
+ return isNormalIPv6 || isIPv4;
+ }
+
+ private int determineServerPort(NiFiProperties props) {
+ return props.getSslPort() != null ? props.getSslPort() : props.getPort();
+ }
+
+ private String determineServerHostname(NiFiProperties props) {
+ if (props.getSslPort() != null) {
+ return props.getProperty(NiFiProperties.WEB_HTTPS_HOST, "localhost");
+ } else {
+ return props.getProperty(NiFiProperties.WEB_HTTP_HOST, "localhost");
+ }
+ }
+
@Override
public void doScope(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
throws IOException, ServletException {
@@ -78,7 +200,7 @@ public class HostHeaderHandler extends ScopedHandler {
nextScope(target, baseRequest, request, response);
}
- private boolean hostHeaderIsValid(String hostHeader) {
+ boolean hostHeaderIsValid(String hostHeader) {
return validHosts.contains(hostHeader.toLowerCase().trim());
}
@@ -91,10 +213,10 @@ public class HostHeaderHandler extends ScopedHandler {
* Returns an error message to the response and marks the request as handled if the host header is not valid.
* Otherwise passes the request on to the next scoped handler.
*
- * @param target the target (not relevant here)
+ * @param target the target (not relevant here)
* @param baseRequest the original request object
- * @param request the request as an HttpServletRequest
- * @param response the current response
+ * @param request the request as an HttpServletRequest
+ * @param response the current response
*/
@Override
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
@@ -111,11 +233,123 @@ public class HostHeaderHandler extends ScopedHandler {
PrintWriter out = response.getWriter();
out.println("<h1>System Error</h1>");
- out.println("<h2>The request contained an invalid host header [" + StringEscapeUtils.escapeHtml4(hostHeader) +
- "] in the request [" + StringEscapeUtils.escapeHtml4(request.getRequestURI()) +
- "]. Check for request manipulation or third-party intercept. </h2>");
+ // TODO: Change to org.apache.commons.text.StringEscapeUtils
+ out.println("<h2>The request contained an invalid host header [<code>" + StringEscapeUtils.escapeHtml4(hostHeader) +
+ "</code>] in the request [<code>" + StringEscapeUtils.escapeHtml4(request.getRequestURI()) +
+ "</code>]. Check for request manipulation or third-party intercept.</h2>");
+ out.println("<h3>Valid host headers are [<code>empty</code>] or: <br/><code>");
+ out.println(printValidHosts());
+ out.println("</code></h3>");
baseRequest.setHandled(true);
}
}
+
+ String printValidHosts() {
+ StringBuilder sb = new StringBuilder("<ul>");
+ for (String vh : validHosts) {
+ if (StringUtils.isNotBlank(vh))
+ sb.append("<li>").append(StringEscapeUtils.escapeHtml4(vh)).append("</li>\n");
+ }
+ return sb.append("</ul>\n").toString();
+ }
+
+ public static List<String> generateDefaultHostnames(NiFiProperties niFiProperties) {
+ List<String> validHosts = new ArrayList<>();
+ int serverPort = 0;
+
+ if (niFiProperties == null) {
+ logger.warn("NiFiProperties not configured; returning minimal default hostnames");
+ } else {
+ try {
+ serverPort = niFiProperties.getConfiguredHttpOrHttpsPort();
+ } catch (RuntimeException e) {
+ logger.warn("Cannot fully generate list of default hostnames because the server port is not configured in nifi.properties. Defaulting to port 0 for host header evaluation");
+ }
+
+ // Add any custom network interfaces
+ try {
+ final int lambdaPort = serverPort;
+ List<String> customIPs = extractIPsFromNetworkInterfaces(niFiProperties);
+ customIPs.stream().forEach(ip -> {
+ validHosts.add(ip);
+ validHosts.add(ip + ":" + lambdaPort);
+ });
+ } catch (final Exception e) {
+ logger.warn("Failed to determine custom network interfaces.", e);
+ }
+ }
+
+ // Sometimes the hostname is left empty but the port is always populated
+ validHosts.add("127.0.0.1");
+ validHosts.add("127.0.0.1:" + serverPort);
+ validHosts.add("localhost");
+ validHosts.add("localhost:" + serverPort);
+ validHosts.add("[::1]");
+ validHosts.add("[::1]:" + serverPort);
+
+ // Add the loopback and actual IP address and hostname used
+ try {
+ validHosts.add(InetAddress.getLoopbackAddress().getHostAddress().toLowerCase());
+ validHosts.add(InetAddress.getLoopbackAddress().getHostAddress().toLowerCase() + ":" + serverPort);
+
+ validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase());
+ validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase() + ":" + serverPort);
+
+ validHosts.add(InetAddress.getLocalHost().getHostAddress().toLowerCase());
+ validHosts.add(InetAddress.getLocalHost().getHostAddress().toLowerCase() + ":" + serverPort);
+ } catch (final Exception e) {
+ logger.warn("Failed to determine local hostname.", e);
+ }
+
+ // Dedupe but maintain order
+ final List<String> uniqueHosts = uniqueList(validHosts);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Determined {} valid default hostnames and IP addresses for incoming headers: {}", new Object[]{uniqueHosts.size(), StringUtils.join(uniqueHosts, ", ")});
+ }
+ return uniqueHosts;
+ }
+
+ /**
+ * Extracts the list of IP addresses from custom bound network interfaces. If both HTTPS and HTTP interfaces are
+ * defined and HTTPS is enabled, only HTTPS interfaces will be returned. If none are defined, an empty list will be
+ * returned.
+ *
+ * @param niFiProperties the NiFiProperties object
+ * @return the list of IP addresses
+ */
+ static List<String> extractIPsFromNetworkInterfaces(NiFiProperties niFiProperties) {
+ Map<String, String> networkInterfaces = niFiProperties.isHTTPSConfigured() ? niFiProperties.getHttpsNetworkInterfaces() : niFiProperties.getHttpNetworkInterfaces();
+ if (isNotDefined(networkInterfaces)) {
+ // No custom interfaces defined
+ return new ArrayList<>(0);
+ } else {
+ List<String> allIPAddresses = new ArrayList<>();
+ for (Map.Entry<String, String> entry : networkInterfaces.entrySet()) {
+ final String networkInterfaceName = entry.getValue();
+ try {
+ NetworkInterface ni = NetworkInterface.getByName(networkInterfaceName);
+ List<String> ipAddresses = Collections.list(ni.getInetAddresses()).stream().map(inetAddress -> inetAddress.getHostAddress().toLowerCase()).collect(Collectors.toList());
+ logger.debug("Resolved the following IP addresses for network interface {}: {}", new Object[]{networkInterfaceName, StringUtils.join(ipAddresses, ", ")});
+ allIPAddresses.addAll(ipAddresses);
+ } catch (SocketException e) {
+ logger.warn("Cannot resolve network interface named " + networkInterfaceName);
+ }
+ }
+
+ // Dedupe while maintaining order
+ return uniqueList(allIPAddresses);
+ }
+ }
+
+ /**
+ * Returns true if the provided map of properties and network interfaces is null, empty, or the actual definitions are empty.
+ *
+ * @param networkInterfaces the map of properties to bindings
+ * ({@code ["nifi.web.http.network.interface.first":"eth0"]})
+ * @return
+ */
+ static boolean isNotDefined(Map<String, String> networkInterfaces) {
+ return networkInterfaces == null || networkInterfaces.isEmpty() || networkInterfaces.values().stream().filter(value -> !Strings.isNullOrEmpty(value)).collect(Collectors.toList()).isEmpty();
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderSanitizationCustomizer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderSanitizationCustomizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderSanitizationCustomizer.java
deleted file mode 100644
index 3a7b22b..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderSanitizationCustomizer.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.web.server;
-
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HostHeaderSanitizationCustomizer implements HttpConfiguration.Customizer {
- private static final Logger logger = LoggerFactory.getLogger(HostHeaderSanitizationCustomizer.class);
-
- private final String serverName;
- private final int serverPort;
- private final List<String> validHosts;
-
- /**
- * @param serverName the {@code serverName} to set on the request (the {@code serverPort} will not be set)
- */
- public HostHeaderSanitizationCustomizer(String serverName) {
- this(serverName, 0);
- }
-
- /**
- * @param serverName the {@code serverName} to set on the request
- * @param serverPort the {@code serverPort} to set on the request
- */
- public HostHeaderSanitizationCustomizer(String serverName, int serverPort) {
- this.serverName = Objects.requireNonNull(serverName);
- this.serverPort = serverPort;
-
- validHosts = new ArrayList<>();
- validHosts.add(serverName.toLowerCase());
- validHosts.add(serverName.toLowerCase() + ":" + serverPort);
- // Sometimes the hostname is left empty but the port is always populated
- validHosts.add("localhost");
- validHosts.add("localhost:" + serverPort);
- try {
- validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase());
- validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase() + ":" + serverPort);
- } catch (final Exception e) {
- logger.warn("Failed to determine local hostname.", e);
- }
-
- logger.info("Created " + this.toString());
- }
-
- @Override
- public void customize(Connector connector, HttpConfiguration channelConfig, Request request) {
- final String hostHeader = request.getHeader("Host");
- logger.debug("Received request [" + request.getRequestURI() + "] with host header: " + hostHeader);
- if (!hostHeaderIsValid(hostHeader)) {
- logger.warn("Request host header [" + hostHeader + "] different from web hostname [" +
- serverName + "(:" + serverPort + ")]. Overriding to [" + serverName + ":" +
- serverPort + request.getRequestURI() + "]");
- request.setAuthority(serverName, serverPort);
- }
- }
-
- private boolean hostHeaderIsValid(String hostHeader) {
- return validHosts.contains(hostHeader.toLowerCase().trim());
- }
-
- @Override
- public String toString() {
- return "HostHeaderSanitizationCustomizer for " + serverName + ":" + serverPort;
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
index 744c127..de53a24 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
@@ -155,28 +155,22 @@ public class JettyServer implements NiFiServer {
// load wars from the bundle
Handler warHandlers = loadWars(bundles);
- // Create a handler for the host header and add it to the server
- String serverName = determineServerHostname();
- int serverPort = determineServerPort();
- HostHeaderHandler hostHeaderHandler = new HostHeaderHandler(serverName, serverPort);
- logger.info("Created HostHeaderHandler [" + hostHeaderHandler.toString() + "]");
-
HandlerList allHandlers = new HandlerList();
- allHandlers.addHandler(hostHeaderHandler);
- allHandlers.addHandler(warHandlers);
- server.setHandler(allHandlers);
- }
- private int determineServerPort() {
- return props.getSslPort() != null ? props.getSslPort() : props.getPort();
- }
+ // Only restrict the host header if running in HTTPS mode
+ if (props.isHTTPSConfigured()) {
+ // Create a handler for the host header and add it to the server
+ HostHeaderHandler hostHeaderHandler = new HostHeaderHandler(props);
+ logger.info("Created HostHeaderHandler [" + hostHeaderHandler.toString() + "]");
- private String determineServerHostname() {
- if (props.getSslPort() != null) {
- return props.getProperty(NiFiProperties.WEB_HTTPS_HOST, "localhost");
+ // Add this before the WAR handlers
+ allHandlers.addHandler(hostHeaderHandler);
} else {
- return props.getProperty(NiFiProperties.WEB_HTTP_HOST, "localhost");
+ logger.info("Running in HTTP mode; host headers not restricted");
}
+
+ allHandlers.addHandler(warHandlers);
+ server.setHandler(allHandlers);
}
private Handler loadWars(final Set<Bundle> bundles) {
@@ -607,8 +601,6 @@ public class JettyServer implements NiFiServer {
httpConfiguration.setRequestHeaderSize(headerSize);
httpConfiguration.setResponseHeaderSize(headerSize);
- addHostHeaderSanitizationCustomizer(httpConfiguration);
-
if (props.getPort() != null) {
final Integer port = props.getPort();
if (port < 0 || (int) Math.pow(2, 16) <= port) {
@@ -710,7 +702,6 @@ public class JettyServer implements NiFiServer {
httpsConfiguration.setSecureScheme("https");
httpsConfiguration.setSecurePort(props.getSslPort());
httpsConfiguration.addCustomizer(new SecureRequestCustomizer());
- addHostHeaderSanitizationCustomizer(httpsConfiguration);
// build the connector
return new ServerConnector(server,
@@ -718,18 +709,6 @@ public class JettyServer implements NiFiServer {
new HttpConnectionFactory(httpsConfiguration));
}
- private void addHostHeaderSanitizationCustomizer(HttpConfiguration httpConfiguration) {
- // Add the HostHeaderCustomizer to the configuration
- HttpConfiguration.Customizer hostHeaderCustomizer;
- if (props.getSslPort() != null) {
- hostHeaderCustomizer = new HostHeaderSanitizationCustomizer(props.getProperty(NiFiProperties.WEB_HTTPS_HOST), props.getSslPort());
- } else {
- hostHeaderCustomizer = new HostHeaderSanitizationCustomizer(props.getProperty(NiFiProperties.WEB_HTTP_HOST), props.getPort());
- }
- httpConfiguration.addCustomizer(hostHeaderCustomizer);
- logger.info("Added HostHeaderSanitizationCustomizer to HttpConfiguration: " + hostHeaderCustomizer);
- }
-
private SslContextFactory createSslContextFactory() {
final SslContextFactory contextFactory = new SslContextFactory();
configureSslContextFactory(contextFactory, props);
http://git-wip-us.apache.org/repos/asf/nifi/blob/bf854bab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/test/java/org/apache/nifi/web/server/HostHeaderHandlerTest.groovy
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/test/java/org/apache/nifi/web/server/HostHeaderHandlerTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/test/java/org/apache/nifi/web/server/HostHeaderHandlerTest.groovy
new file mode 100644
index 0000000..a08be9d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/test/java/org/apache/nifi/web/server/HostHeaderHandlerTest.groovy
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.server
+
+import org.apache.commons.lang3.StringUtils
+import org.apache.nifi.properties.StandardNiFiProperties
+import org.apache.nifi.util.NiFiProperties
+import org.junit.After
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+@RunWith(JUnit4.class)
+class HostHeaderHandlerTest extends GroovyTestCase {
+ private static final Logger logger = LoggerFactory.getLogger(HostHeaderHandlerTest.class)
+
+ private static final String DEFAULT_HOSTNAME = "nifi.apache.org"
+ private static final String ACTUAL_HOSTNAME = InetAddress.getLocalHost().getHostName().toLowerCase()
+ private static final int DEFAULT_PORT = 8080
+ private static final List<String> DEFAULT_HOSTS_1_5_0 = [DEFAULT_HOSTNAME, "localhost", ACTUAL_HOSTNAME]
+ private static
+ final List<String> DEFAULT_HOSTS_AND_PORTS_1_5_0 = DEFAULT_HOSTS_1_5_0.collectMany { it -> [it, "${it}:${DEFAULT_PORT}"] }
+
+ // Post 1.5.0 list
+ private static final String ACTUAL_IP = InetAddress.getLocalHost().getHostAddress()
+ private static final String LOOPBACK_IP = InetAddress.getLoopbackAddress().getHostAddress()
+ private static
+ final List<String> DEFAULT_HOSTS = DEFAULT_HOSTS_1_5_0 - DEFAULT_HOSTNAME + ["[::1]", "127.0.0.1", ACTUAL_IP, LOOPBACK_IP]
+ private static
+ final List<String> DEFAULT_HOSTS_AND_PORTS = DEFAULT_HOSTS.collectMany { it -> [it, "${it}:${DEFAULT_PORT}"] }
+
+ @BeforeClass
+ static void setUpOnce() throws Exception {
+ logger.metaClass.methodMissing = { String name, args ->
+ logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+ }
+ }
+
+ @Before
+ void setUp() throws Exception {
+ }
+
+ @After
+ void tearDown() throws Exception {
+ }
+
+ @Test
+ void testConstructorShouldAcceptSingleValues() throws Exception {
+ // Arrange
+ String hostname = DEFAULT_HOSTNAME
+ int port = DEFAULT_PORT
+ logger.info("Hostname: ${hostname} | port: ${port}")
+
+ // Act
+ HostHeaderHandler handler = new HostHeaderHandler(hostname, port)
+ logger.info("Handler: ${handler}")
+
+ // Assert
+ assert handler.hostHeaderIsValid(hostname)
+ assert handler.hostHeaderIsValid("${hostname}:${port}")
+ }
+
+ /**
+ * The feature was introduced in Apache NiFi 1.5.0 but the behavior was changed following that release to include the actual IP address of the server, IPv6 ::1, and 127.0.0.1.
+ * @throws Exception
+ */
+ @Test
+ void testShouldHandle_1_5_0_DefaultValues() throws Exception {
+ // Arrange
+ String hostname = DEFAULT_HOSTNAME
+ int port = DEFAULT_PORT
+ logger.info("Hostname: ${hostname} | port: ${port}")
+
+ // Act
+ HostHeaderHandler handler = new HostHeaderHandler(hostname, port)
+ logger.info("Handler: ${handler}")
+
+ // Assert
+ DEFAULT_HOSTS_AND_PORTS_1_5_0.each { String host ->
+ logger.debug("Validating ${host}")
+ assert handler.hostHeaderIsValid(host)
+ }
+ }
+
+ @Test
+ void testNewConstructorShouldHandleCurrentDefaultValues() throws Exception {
+ // Arrange
+ String hostname = DEFAULT_HOSTNAME
+ int port = DEFAULT_PORT
+ logger.info("Hostname: ${hostname} | port: ${port}")
+
+ Properties rawProps = new Properties()
+ rawProps.putAll([
+ (NiFiProperties.WEB_HTTPS_HOST): DEFAULT_HOSTNAME,
+ (NiFiProperties.WEB_HTTPS_PORT): "${DEFAULT_PORT}".toString(),
+ ])
+ NiFiProperties simpleProperties = new StandardNiFiProperties(rawProps)
+
+ // Act
+ HostHeaderHandler handler = new HostHeaderHandler(simpleProperties)
+ logger.info("Handler: ${handler}")
+
+ // Assert
+ DEFAULT_HOSTS_AND_PORTS.each { String host ->
+ logger.debug("Validating ${host}")
+ assert handler.hostHeaderIsValid(host)
+ }
+ }
+
+ @Test
+ void testShouldParseCustomHostnames() throws Exception {
+ // Arrange
+ String hostname = DEFAULT_HOSTNAME
+ int port = DEFAULT_PORT
+ logger.info("Hostname: ${hostname} | port: ${port}")
+
+ List<String> otherHosts = ["someotherhost.com:9999", "yetanotherbadhost.com", "10.10.10.1:1234", "100.100.100.1"]
+ String concatenatedHosts = otherHosts.join(",")
+
+ Properties rawProps = new Properties()
+ rawProps.putAll([
+ (NiFiProperties.WEB_HTTPS_HOST): DEFAULT_HOSTNAME,
+ (NiFiProperties.WEB_HTTPS_PORT): "${DEFAULT_PORT}".toString(),
+ (NiFiProperties.WEB_PROXY_HOST): concatenatedHosts
+ ])
+ NiFiProperties simpleProperties = new StandardNiFiProperties(rawProps)
+
+ HostHeaderHandler handler = new HostHeaderHandler(simpleProperties)
+ logger.info("Handler: ${handler}")
+
+ // Act
+ List<String> customHostnames = handler.parseCustomHostnames(simpleProperties)
+ logger.info("Parsed custom hostnames: ${customHostnames}")
+
+ // Assert
+ assert customHostnames.size() == otherHosts.size() + 2 // Two provided hostnames had ports
+ otherHosts.each { String host ->
+ logger.debug("Checking ${host}")
+ assert customHostnames.contains(host)
+ String portlessHost = "${host.split(":", 2)[0]}".toString()
+ logger.debug("Checking ${portlessHost}")
+ assert customHostnames.contains(portlessHost)
+ }
+ }
+
+ @Test
+ void testParseCustomHostnamesShouldHandleIPv6WithoutPorts() throws Exception {
+ // Arrange
+ String hostname = DEFAULT_HOSTNAME
+ int port = DEFAULT_PORT
+ logger.info("Hostname: ${hostname} | port: ${port}")
+
+ List<String> ipv6Hosts = ["ABCD:EF01:2345:6789:ABCD:EF01:2345:6789",
+ "2001:DB8:0:0:8:800:200C:417A",
+ "FF01:0:0:0:0:0:0:101",
+ "0:0:0:0:0:0:0:1",
+ "0:0:0:0:0:0:0:0",
+ "2001:DB8::8:800:200C:417A",
+ "FF01::101",
+ "::1",
+ "::",
+ "0:0:0:0:0:0:13.1.68.3",
+ "0:0:0:0:0:FFFF:129.144.52.38",
+ "::13.1.68.3",
+ "FFFF:129.144.52.38",
+ "::FFFF:129.144.52.38"]
+ String concatenatedHosts = ipv6Hosts.join(",")
+
+ Properties rawProps = new Properties()
+ rawProps.putAll([
+ (NiFiProperties.WEB_HTTPS_HOST): DEFAULT_HOSTNAME,
+ (NiFiProperties.WEB_HTTPS_PORT): "${DEFAULT_PORT}".toString(),
+ (NiFiProperties.WEB_PROXY_HOST): concatenatedHosts
+ ])
+ NiFiProperties simpleProperties = new StandardNiFiProperties(rawProps)
+
+ HostHeaderHandler handler = new HostHeaderHandler(simpleProperties)
+ logger.info("Handler: ${handler}")
+
+ // Act
+ List<String> customHostnames = handler.parseCustomHostnames(simpleProperties)
+ logger.info("Parsed custom hostnames: ${customHostnames}")
+
+ // Assert
+ assert customHostnames.size() == ipv6Hosts.size()
+ ipv6Hosts.each { String host ->
+ logger.debug("Checking ${host}")
+ assert customHostnames.contains(host)
+ }
+ }
+
+ @Test
+ void testParseCustomHostnamesShouldHandleIPv6WithPorts() throws Exception {
+ // Arrange
+ String hostname = DEFAULT_HOSTNAME
+ int port = DEFAULT_PORT
+ logger.info("Hostname: ${hostname} | port: ${port}")
+
+ List<String> ipv6Hosts = ["[ABCD:EF01:2345:6789:ABCD:EF01:2345:6789]:1234",
+ "[2001:DB8:0:0:8:800:200C:417A]:1234",
+ "[FF01:0:0:0:0:0:0:101]:1234",
+ "[0:0:0:0:0:0:0:1]:1234",
+ "[0:0:0:0:0:0:0:0]:1234",
+ "[2001:DB8::8:800:200C:417A]:1234",
+ "[FF01::101]:1234",
+ "[::1]:1234",
+ "[::]:1234",
+ "[0:0:0:0:0:0:13.1.68.3]:1234",
+ "[0:0:0:0:0:FFFF:129.144.52.38]:1234",
+ "[::13.1.68.3]:1234",
+ "[FFFF:129.144.52.38]:1234",
+ "[::FFFF:129.144.52.38]:1234"]
+ String concatenatedHosts = ipv6Hosts.join(",")
+
+ Properties rawProps = new Properties()
+ rawProps.putAll([
+ (NiFiProperties.WEB_HTTPS_HOST): DEFAULT_HOSTNAME,
+ (NiFiProperties.WEB_HTTPS_PORT): "${DEFAULT_PORT}".toString(),
+ (NiFiProperties.WEB_PROXY_HOST): concatenatedHosts
+ ])
+ NiFiProperties simpleProperties = new StandardNiFiProperties(rawProps)
+
+ HostHeaderHandler handler = new HostHeaderHandler(simpleProperties)
+ logger.info("Handler: ${handler}")
+
+ // Act
+ List<String> customHostnames = handler.parseCustomHostnames(simpleProperties)
+ logger.info("Parsed custom hostnames: ${customHostnames}")
+
+ // Assert
+ assert customHostnames.size() == ipv6Hosts.size() * 2
+ ipv6Hosts.each { String host ->
+ logger.debug("Checking ${host}")
+ assert customHostnames.contains(host)
+ String portlessHost = "${StringUtils.substringBeforeLast(host, ":")}".toString()
+ logger.debug("Checking ${portlessHost}")
+ assert customHostnames.contains(portlessHost)
+ }
+ }
+
+ @Test
+ void testShouldIdentifyIPv6Addresses() throws Exception {
+ // Arrange
+ List<String> ipv6Hosts = ["ABCD:EF01:2345:6789:ABCD:EF01:2345:6789",
+ "2001:DB8:0:0:8:800:200C:417A",
+ "FF01:0:0:0:0:0:0:101",
+ "0:0:0:0:0:0:0:1",
+ "0:0:0:0:0:0:0:0",
+ "2001:DB8::8:800:200C:417A",
+ "FF01::101",
+ "::1",
+ "::",
+ "0:0:0:0:0:0:13.1.68.3",
+ "0:0:0:0:0:FFFF:129.144.52.38",
+ "::13.1.68.3",
+ "FFFF:129.144.52.38",
+ "::FFFF:129.144.52.38"]
+
+ // Act
+ List<Boolean> hostsAreIPv6 = ipv6Hosts.collect { String host ->
+ boolean isIPv6 = HostHeaderHandler.isIPv6Address(host)
+ logger.info("Hostname is IPv6: ${host} | ${isIPv6}")
+ isIPv6
+ }
+
+ // Assert
+ assert hostsAreIPv6.every()
+ }
+}
[17/50] nifi git commit: NIFI-4766: - Documenting nifi.web.proxy.host
property. - Updating proxy confiugration. NIFI-4618: - Documenting
nifi.web.proxy.context.path property. - Addressing PR feedback. - This closes
#2410
Posted by al...@apache.org.
NIFI-4766:
- Documenting nifi.web.proxy.host property.
- Updating proxy confiugration.
NIFI-4618:
- Documenting nifi.web.proxy.context.path property.
- Addressing PR feedback.
- This closes #2410
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c582d869
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c582d869
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c582d869
Branch: refs/heads/HDF-3.1-maint
Commit: c582d8692842225364a0b9d55a54d7b5f35704de
Parents: bf854ba
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Jan 17 15:55:40 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Jan 19 11:06:54 2018 -0500
----------------------------------------------------------------------
.../src/main/asciidoc/administration-guide.adoc | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/c582d869/nifi-docs/src/main/asciidoc/administration-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 6bbf8a2..abe520f 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -2645,6 +2645,17 @@ documentation of the proxy for guidance for your deployment environment and use
...
....
+* Additional NiFi proxy configuration must be updated to allow expected Host and context paths HTTP headers.
+
+** By default, if NiFi is running securely it will only accept HTTP requests with a Host header matching the host[:port] that it is bound to. If NiFi is to accept requests directed to a different
+host[:port] the expected values need to be configured. This may be required when running behind a proxy or in a containerized environment. This is configured in a comma
+separated list in _nifi.properties_ using the `nifi.web.proxy.host` property (e.g. localhost:18443, proxyhost:443). IPv6 addressed are accepted. Please refer to
+RFC 5952 Sections link:https://tools.ietf.org/html/rfc5952#section-4[4] and link:https://tools.ietf.org/html/rfc5952#section-6[6] for additional details.
+
+** NiFi will only accept HTTP requests with a X-ProxyContextPath or X-Forwarded-Context header if the value is whitelisted in the `nifi.web.proxy.context.path` property in
+_nifi.properties_. This property accepts a comma separated list of expected values. In the event an incoming request has an X-ProxyContextPath or X-Forwarded-Context header value that is not
+present in the whitelist, the "An unexpected error has occurred" page will be shown and an error will be written to the nifi-app.log.
+
[[kerberos_service]]
== Kerberos Service
NiFi can be configured to use Kerberos SPNEGO (or "Kerberos Service") for authentication. In this scenario, users will hit the REST endpoint `/access/kerberos` and the server will respond with a `401` status code and the challenge response header `WWW-Authenticate: Negotiate`. This communicates to the browser to use the GSS-API and load the user's Kerberos ticket and provide it as a Base64-encoded header value in the subsequent request. It will be of the form `Authorization: Negotiate YII...`. NiFi will attempt to validate this ticket with the KDC. If it is successful, the user's _principal_ will be returned as the identity, and the flow will follow login/credential authentication, in that a JWT will be issued in the response to prevent the unnecessary overhead of Kerberos authentication on every subsequent request. If the ticket cannot be validated, it will return with the appropriate error response code. The user will then be able to provide their Kerberos credentials to the login
form if the `KerberosLoginIdentityProvider` has been configured. See <<kerberos_login_identity_provider>> login identity provider for more details.
@@ -3083,6 +3094,11 @@ Providing three total network interfaces, including `nifi.web.https.network.int
|nifi.web.jetty.working.directory|The location of the Jetty working directory. The default value is `./work/jetty`.
|nifi.web.jetty.threads|The number of Jetty threads. The default value is `200`.
|nifi.web.max.header.size|The maximum size allowed for request and response headers. The default value is 16 KB.
+|nifi.web.proxy.host|A comma separated list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to.
+For example, when running in a Docker container or behind a proxy (e.g. localhost:18443, proxyhost:443). By default, this value is blank meaning NiFi should only allow requests sent to the
+host[:port] that NiFi is bound to.
+|nifi.web.proxy.context.path|A comma separated list of allowed HTTP X-ProxyContextPath or X-Forwarded-Context header values to consider. By default, this value is
+blank meaning all requests containing a proxy context path are rejected. Configuring this property would allow requests where the proxy path is contained in this listing.
|====
=== Security Properties
[12/50] nifi git commit: NIFI-4763: Ignore differences in components'
Bundle Version when comparing a local flow to a flow in the registry
Posted by al...@apache.org.
NIFI-4763: Ignore differences in components' Bundle Version when comparing a local flow to a flow in the registry
This closes #2393.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/fbc4ec36
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/fbc4ec36
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/fbc4ec36
Branch: refs/heads/HDF-3.1-maint
Commit: fbc4ec3616ed3794fd94675e6d4dbf65d846894e
Parents: e77e66e
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Jan 10 10:01:27 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 15:02:38 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/groups/StandardProcessGroup.java | 9 ++++++++-
.../java/org/apache/nifi/web/StandardNiFiServiceFacade.java | 6 ++++++
.../main/java/org/apache/nifi/web/api/dto/DtoFactory.java | 6 ++++++
3 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/fbc4ec36/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index 4cd6e2a..baf02bc 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -3252,6 +3252,11 @@ public final class StandardProcessGroup implements ProcessGroup {
final Set<String> updatedVersionedComponentIds = new HashSet<>();
for (final FlowDifference diff : flowComparison.getDifferences()) {
+ // Ignore these as local differences for now because we can't do anything with it
+ if (diff.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
+ continue;
+ }
+
// If this update adds a new Controller Service, then we need to check if the service already exists at a higher level
// and if so compare our VersionedControllerService to the existing service.
if (diff.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
@@ -4187,7 +4192,9 @@ public final class StandardProcessGroup implements ProcessGroup {
final FlowComparator flowComparator = new StandardFlowComparator(snapshotFlow, currentFlow, getAncestorGroupServiceIds(), new EvolvingDifferenceDescriptor());
final FlowComparison comparison = flowComparator.compare();
- final Set<FlowDifference> differences = comparison.getDifferences();
+ final Set<FlowDifference> differences = comparison.getDifferences().stream()
+ .filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
+ .collect(Collectors.toCollection(HashSet::new));
LOG.debug("There are {} differences between this Local Flow and the Versioned Flow: {}", differences.size(), differences);
return differences;
http://git-wip-us.apache.org/repos/asf/nifi/blob/fbc4ec36/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index 3df75bb..65f6329 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -3964,6 +3964,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream()
.filter(difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED) // components that are added are not components that will be affected in the local flow.
+ .filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
.map(difference -> {
final VersionedComponent localComponent = difference.getComponentA();
@@ -3995,6 +3996,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
.collect(Collectors.toCollection(HashSet::new));
for (final FlowDifference difference : comparison.getDifferences()) {
+ // Ignore these as local differences for now because we can't do anything with it
+ if (difference.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
+ continue;
+ }
+
final VersionedComponent localComponent = difference.getComponentA();
if (localComponent == null) {
continue;
http://git-wip-us.apache.org/repos/asf/nifi/blob/fbc4ec36/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 3d9f521..36da701 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -118,6 +118,7 @@ import org.apache.nifi.registry.flow.VersionControlInformation;
import org.apache.nifi.registry.flow.VersionedComponent;
import org.apache.nifi.registry.flow.VersionedFlowState;
import org.apache.nifi.registry.flow.VersionedFlowStatus;
+import org.apache.nifi.registry.flow.diff.DifferenceType;
import org.apache.nifi.registry.flow.diff.FlowComparison;
import org.apache.nifi.registry.flow.diff.FlowDifference;
import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedComponent;
@@ -2219,6 +2220,11 @@ public final class DtoFactory {
final Map<ComponentDifferenceDTO, List<DifferenceDTO>> differencesByComponent = new HashMap<>();
for (final FlowDifference difference : comparison.getDifferences()) {
+ // Ignore these as local differences for now because we can't do anything with it
+ if (difference.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
+ continue;
+ }
+
final ComponentDifferenceDTO componentDiff = createComponentDifference(difference);
final List<DifferenceDTO> differences = differencesByComponent.computeIfAbsent(componentDiff, key -> new ArrayList<>());
[25/50] nifi git commit: NIFI-4773: Moved DB Fetch processors'
connection code from setup to onTrigger
Posted by al...@apache.org.
NIFI-4773: Moved DB Fetch processors' connection code from setup to onTrigger
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #2422.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/0ddd0b19
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/0ddd0b19
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/0ddd0b19
Branch: refs/heads/HDF-3.1-maint
Commit: 0ddd0b1955b55a0b6342628908c637b5f7d30409
Parents: 4829f51
Author: Matthew Burgess <ma...@apache.org>
Authored: Mon Jan 22 13:42:35 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 09:00:12 2018 -0500
----------------------------------------------------------------------
.../AbstractDatabaseFetchProcessor.java | 85 ++++++++++++--------
.../processors/standard/GenerateTableFetch.java | 7 +-
.../processors/standard/QueryDatabaseTable.java | 9 ++-
3 files changed, 61 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/0ddd0b19/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
index fa2a86e..2145929 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
@@ -50,6 +50,7 @@ import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import static java.sql.Types.ARRAY;
import static java.sql.Types.BIGINT;
@@ -176,6 +177,10 @@ public abstract class AbstractDatabaseFetchProcessor extends AbstractSessionFact
// pre-fetched when the processor is scheduled, rather than having to populate them on-the-fly.
protected volatile boolean isDynamicMaxValues = false;
+ // This value is cleared when the processor is scheduled, and set to true after setup() is called and completes successfully. This enables
+ // the setup logic to be performed in onTrigger() versus OnScheduled to avoid any issues with DB connection when first scheduled to run.
+ protected final AtomicBoolean setupComplete = new AtomicBoolean(false);
+
private static SimpleDateFormat TIME_TYPE_FORMAT = new SimpleDateFormat("HH:mm:ss.SSS");
// A Map (name to value) of initial maximum-value properties, filled at schedule-time and used at trigger-time
@@ -222,49 +227,61 @@ public abstract class AbstractDatabaseFetchProcessor extends AbstractSessionFact
return super.customValidate(validationContext);
}
+ @Override
+ public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
+ // If the max-value columns have changed, we need to re-fetch the column info from the DB
+ if (MAX_VALUE_COLUMN_NAMES.equals(descriptor) && newValue != null && !newValue.equals(oldValue)) {
+ setupComplete.set(false);
+ }
+ }
+
public void setup(final ProcessContext context) {
setup(context,true,null);
}
public void setup(final ProcessContext context, boolean shouldCleanCache, FlowFile flowFile) {
- final String maxValueColumnNames = context.getProperty(MAX_VALUE_COLUMN_NAMES).evaluateAttributeExpressions(flowFile).getValue();
+ synchronized (setupComplete) {
+ setupComplete.set(false);
+ final String maxValueColumnNames = context.getProperty(MAX_VALUE_COLUMN_NAMES).evaluateAttributeExpressions(flowFile).getValue();
- // If there are no max-value column names specified, we don't need to perform this processing
- if (StringUtils.isEmpty(maxValueColumnNames)) {
- return;
- }
+ // If there are no max-value column names specified, we don't need to perform this processing
+ if (StringUtils.isEmpty(maxValueColumnNames)) {
+ return;
+ }
- // Try to fill the columnTypeMap with the types of the desired max-value columns
- final DBCPService dbcpService = context.getProperty(DBCP_SERVICE).asControllerService(DBCPService.class);
- final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
-
- final DatabaseAdapter dbAdapter = dbAdapters.get(context.getProperty(DB_TYPE).getValue());
- try (final Connection con = dbcpService.getConnection();
- final Statement st = con.createStatement()) {
-
- // Try a query that returns no rows, for the purposes of getting metadata about the columns. It is possible
- // to use DatabaseMetaData.getColumns(), but not all drivers support this, notably the schema-on-read
- // approach as in Apache Drill
- String query = dbAdapter.getSelectStatement(tableName, maxValueColumnNames, "1 = 0", null, null, null);
- ResultSet resultSet = st.executeQuery(query);
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- int numCols = resultSetMetaData.getColumnCount();
- if (numCols > 0) {
- if (shouldCleanCache){
- columnTypeMap.clear();
- }
- for (int i = 1; i <= numCols; i++) {
- String colName = resultSetMetaData.getColumnName(i).toLowerCase();
- String colKey = getStateKey(tableName, colName);
- int colType = resultSetMetaData.getColumnType(i);
- columnTypeMap.putIfAbsent(colKey, colType);
+ // Try to fill the columnTypeMap with the types of the desired max-value columns
+ final DBCPService dbcpService = context.getProperty(DBCP_SERVICE).asControllerService(DBCPService.class);
+ final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
+
+ final DatabaseAdapter dbAdapter = dbAdapters.get(context.getProperty(DB_TYPE).getValue());
+ try (final Connection con = dbcpService.getConnection();
+ final Statement st = con.createStatement()) {
+
+ // Try a query that returns no rows, for the purposes of getting metadata about the columns. It is possible
+ // to use DatabaseMetaData.getColumns(), but not all drivers support this, notably the schema-on-read
+ // approach as in Apache Drill
+ String query = dbAdapter.getSelectStatement(tableName, maxValueColumnNames, "1 = 0", null, null, null);
+ ResultSet resultSet = st.executeQuery(query);
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ int numCols = resultSetMetaData.getColumnCount();
+ if (numCols > 0) {
+ if (shouldCleanCache) {
+ columnTypeMap.clear();
+ }
+ for (int i = 1; i <= numCols; i++) {
+ String colName = resultSetMetaData.getColumnName(i).toLowerCase();
+ String colKey = getStateKey(tableName, colName);
+ int colType = resultSetMetaData.getColumnType(i);
+ columnTypeMap.putIfAbsent(colKey, colType);
+ }
+ } else {
+ throw new ProcessException("No columns found in table from those specified: " + maxValueColumnNames);
}
- } else {
- throw new ProcessException("No columns found in table from those specified: " + maxValueColumnNames);
- }
- } catch (SQLException e) {
- throw new ProcessException("Unable to communicate with database in order to determine column types", e);
+ } catch (SQLException e) {
+ throw new ProcessException("Unable to communicate with database in order to determine column types", e);
+ }
+ setupComplete.set(true);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/0ddd0b19/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
index 1bae371..8f535b3 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
@@ -155,9 +155,6 @@ public class GenerateTableFetch extends AbstractDatabaseFetchProcessor {
@OnScheduled
public void setup(final ProcessContext context) {
maxValueProperties = getDefaultMaxValueProperties(context.getProperties());
- if (!isDynamicTableName && !isDynamicMaxValues) {
- super.setup(context);
- }
if (context.hasIncomingConnection() && !context.hasNonLoopConnection()) {
getLogger().error("The failure relationship can be used only if there is another incoming connection to this processor.");
}
@@ -165,6 +162,10 @@ public class GenerateTableFetch extends AbstractDatabaseFetchProcessor {
@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
+ // Fetch the column/table info once (if the table name and max value columns are not dynamic). Otherwise do the setup later
+ if (!isDynamicTableName && !isDynamicMaxValues && !setupComplete.get()) {
+ super.setup(context);
+ }
ProcessSession session = sessionFactory.createSession();
FlowFile fileToProcess = null;
http://git-wip-us.apache.org/repos/asf/nifi/blob/0ddd0b19/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
index 0532b79..ffe5845 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
@@ -18,10 +18,10 @@ package org.apache.nifi.processors.standard;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperty;
-import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.Stateful;
+import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
@@ -73,7 +73,7 @@ import static org.apache.nifi.processors.standard.util.JdbcCommon.NORMALIZE_NAME
import static org.apache.nifi.processors.standard.util.JdbcCommon.USE_AVRO_LOGICAL_TYPES;
-@EventDriven
+@TriggerSerially
@InputRequirement(Requirement.INPUT_FORBIDDEN)
@Tags({"sql", "select", "jdbc", "query", "database"})
@SeeAlso({GenerateTableFetch.class, ExecuteSQL.class})
@@ -178,11 +178,14 @@ public class QueryDatabaseTable extends AbstractDatabaseFetchProcessor {
@OnScheduled
public void setup(final ProcessContext context) {
maxValueProperties = getDefaultMaxValueProperties(context.getProperties());
- super.setup(context);
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
+ // Fetch the column/table info once
+ if (!setupComplete.get()) {
+ super.setup(context);
+ }
ProcessSession session = sessionFactory.createSession();
final List<FlowFile> resultSetFlowFiles = new ArrayList<>();
[31/50] nifi git commit: NIFI-4846: AvroTypeUtil to support more
input types for logical decimal conversion
Posted by al...@apache.org.
NIFI-4846: AvroTypeUtil to support more input types for logical decimal conversion
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2451
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d17b9978
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d17b9978
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d17b9978
Branch: refs/heads/HDF-3.1-maint
Commit: d17b99785e508cf7d3d3b7a57d643fe5f2660a4d
Parents: 6d84478
Author: Koji Kawamura <ij...@apache.org>
Authored: Tue Feb 6 16:36:15 2018 +0900
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 20:23:32 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 11 +++++++
.../org/apache/nifi/avro/TestAvroTypeUtil.java | 30 ++++++++++++++++++--
2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d17b9978/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index f7b8d9b..c4f69d4 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -533,8 +533,19 @@ public class AvroTypeUtil {
final BigDecimal rawDecimal;
if (rawValue instanceof BigDecimal) {
rawDecimal = (BigDecimal) rawValue;
+
} else if (rawValue instanceof Double) {
rawDecimal = BigDecimal.valueOf((Double) rawValue);
+
+ } else if (rawValue instanceof String) {
+ rawDecimal = new BigDecimal((String) rawValue);
+
+ } else if (rawValue instanceof Integer) {
+ rawDecimal = new BigDecimal((Integer) rawValue);
+
+ } else if (rawValue instanceof Long) {
+ rawDecimal = new BigDecimal((Long) rawValue);
+
} else {
throw new IllegalTypeConversionException("Cannot convert value " + rawValue + " of type " + rawValue.getClass() + " to a logical decimal");
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d17b9978/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
index d644f7c..d24488e 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
@@ -19,6 +19,7 @@ package org.apache.nifi.avro;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.math.BigDecimal;
@@ -291,14 +292,39 @@ public class TestAvroTypeUtil {
expects.put(new BigDecimal("0.123456789012345678"), "0.12345679");
+ // String to BigDecimal
+ expects.put("123", "123.00000000");
+ expects.put("1234567890.12345678", "1234567890.12345678");
+ expects.put("123456789012345678", "123456789012345678.00000000");
+ expects.put("0.1234567890123456", "0.12345679");
+ expects.put("Not a number", "java.lang.NumberFormatException");
+
+ // Integer to BigDecimal
+ expects.put(123, "123.00000000");
+ expects.put(-1234567, "-1234567.00000000");
+
+ // Long to BigDecimal
+ expects.put(123L, "123.00000000");
+ expects.put(123456789012345678L, "123456789012345678.00000000");
+
expects.forEach((rawValue, expect) -> {
- final Object convertedValue = AvroTypeUtil.convertToAvroObject(rawValue, fieldSchema);
+ final Object convertedValue;
+ try {
+ convertedValue = AvroTypeUtil.convertToAvroObject(rawValue, fieldSchema);
+ } catch (Exception e) {
+ if (expect.equals(e.getClass().getCanonicalName())) {
+ // Expected behavior.
+ return;
+ }
+ fail(String.format("Unexpected exception, %s with %s %s while expecting %s", e, rawValue.getClass().getSimpleName(), rawValue, expect));
+ return;
+ }
assertTrue(convertedValue instanceof ByteBuffer);
final ByteBuffer serializedBytes = (ByteBuffer) convertedValue;
final BigDecimal bigDecimal = new Conversions.DecimalConversion().fromBytes(serializedBytes, fieldSchema, decimalType);
- assertEquals(String.format("%s should be converted to %s", rawValue, expect), expect, bigDecimal.toString());
+ assertEquals(String.format("%s %s should be converted to %s", rawValue.getClass().getSimpleName(), rawValue, expect), expect, bigDecimal.toString());
});
}
[07/50] nifi git commit: NIFI-4754: - Only showing the import link in
the new Process Group dialog when initiating from the toolbar. This closes
#2395.
Posted by al...@apache.org.
NIFI-4754:
- Only showing the import link in the new Process Group dialog when initiating from the toolbar.
This closes #2395.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/cadcd1dc
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/cadcd1dc
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/cadcd1dc
Branch: refs/heads/HDF-3.1-maint
Commit: cadcd1dc78451c31810ad64eb9f3b73357fb04e8
Parents: 658ce50
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Jan 10 15:30:05 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 14:55:24 2018 -0500
----------------------------------------------------------------------
.../header/components/nf-ng-group-component.js | 17 +++++++++--------
.../src/main/webapp/js/nf/canvas/nf-actions.js | 2 +-
2 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/cadcd1dc/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-group-component.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-group-component.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-group-component.js
index 2ce8438..0458b87 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-group-component.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/header/components/nf-ng-group-component.js
@@ -152,12 +152,6 @@
* Show the modal.
*/
show: function () {
- if (nfCommon.canVersionFlows()) {
- $('#import-process-group-link').show();
- } else {
- $('#import-process-group-link').hide();
- }
-
this.getElement().modal('show');
},
@@ -211,7 +205,7 @@
* @argument {object} pt The point that the component was dropped.
*/
dropHandler: function (pt) {
- this.promptForGroupName(pt);
+ this.promptForGroupName(pt, true);
},
/**
@@ -228,8 +222,9 @@
* Prompts the user to enter the name for the group.
*
* @argument {object} pt The point that the group was dropped.
+ * @argument {boolean} showImportLink Whether we should show the import link
*/
- promptForGroupName: function (pt) {
+ promptForGroupName: function (pt, showImportLink) {
var groupComponent = this;
return $.Deferred(function (deferred) {
var addGroup = function () {
@@ -286,6 +281,12 @@
}
}]);
+ if (showImportLink === true && nfCommon.canVersionFlows()) {
+ $('#import-process-group-link').show();
+ } else {
+ $('#import-process-group-link').hide();
+ }
+
// show the dialog
groupComponent.modal.storePt(pt);
groupComponent.modal.show();
http://git-wip-us.apache.org/repos/asf/nifi/blob/cadcd1dc/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index 8a17291..64b0eab 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -1535,7 +1535,7 @@
var origin = nfCanvasUtils.getOrigin(selection);
var pt = {'x': origin.x, 'y': origin.y};
- $.when(nfNgBridge.injector.get('groupComponent').promptForGroupName(pt)).done(function (processGroup) {
+ $.when(nfNgBridge.injector.get('groupComponent').promptForGroupName(pt, false)).done(function (processGroup) {
var group = d3.select('#id-' + processGroup.id);
nfCanvasUtils.moveComponents(selection, group);
});
[48/50] nifi git commit: NIFI-5154: When Processor or Controller
Service is added to a Process Group,
remove any references from it to any other Controller Service that is not
reachable from the newly assigned Process Group Fixed bug in unit test
Address
Posted by al...@apache.org.
NIFI-5154: When Processor or Controller Service is added to a Process Group, remove any references from it to any other Controller Service that is not reachable from the newly assigned Process Group
Fixed bug in unit test
Addressed review feedback/addressed issue where if a group is moved inside another group, the descendant processors of the moved group did not have their service references updated properly. Also addressed an issue where if a service is defined in Group A, then Group B lives within Group A and has a processor that references a service at the level of Group A, we allowed user to move Group B outside of Group A (but wouldn't allow the processor to be moved out of scope by itself).
This closes #2678
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/da94eaca
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/da94eaca
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/da94eaca
Branch: refs/heads/HDF-3.1-maint
Commit: da94eacae7bc31df07d2fbc13c44ff5b48637a53
Parents: fd8f1bf
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri May 4 14:48:53 2018 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Thu May 10 11:12:40 2018 -0400
----------------------------------------------------------------------
.../nifi/groups/StandardProcessGroup.java | 63 +++++++++++++++++++-
.../TestStandardControllerServiceProvider.java | 12 +++-
2 files changed, 70 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/da94eaca/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index f0b7b3e..6170324 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -660,6 +660,10 @@ public final class StandardProcessGroup implements ProcessGroup {
processGroups.put(Objects.requireNonNull(group).getIdentifier(), group);
flowController.onProcessGroupAdded(group);
+
+ group.findAllControllerServices().stream().forEach(this::updateControllerServiceReferences);
+ group.findAllProcessors().stream().forEach(this::updateControllerServiceReferences);
+
onComponentModified();
} finally {
writeLock.unlock();
@@ -829,12 +833,49 @@ public final class StandardProcessGroup implements ProcessGroup {
processor.getVariableRegistry().setParent(getVariableRegistry());
processors.put(processorId, processor);
flowController.onProcessorAdded(processor);
+ updateControllerServiceReferences(processor);
onComponentModified();
} finally {
writeLock.unlock();
}
}
+ /**
+ * Looks for any property that is configured on the given component that references a Controller Service.
+ * If any exists, and that Controller Service is not accessible from this Process Group, then the given
+ * component will be removed from the service's referencing components.
+ *
+ * @param component the component whose invalid references should be removed
+ */
+ private void updateControllerServiceReferences(final ConfiguredComponent component) {
+ for (final Map.Entry<PropertyDescriptor, String> entry : component.getProperties().entrySet()) {
+ final String serviceId = entry.getValue();
+ if (serviceId == null) {
+ continue;
+ }
+
+ final PropertyDescriptor propertyDescriptor = entry.getKey();
+ final Class<? extends ControllerService> serviceClass = propertyDescriptor.getControllerServiceDefinition();
+
+ if (serviceClass != null) {
+ final boolean validReference = isValidServiceReference(serviceId, serviceClass);
+ final ControllerServiceNode serviceNode = controllerServiceProvider.getControllerServiceNode(serviceId);
+ if (serviceNode != null) {
+ if (validReference) {
+ serviceNode.addReference(component);
+ } else {
+ serviceNode.removeReference(component);
+ }
+ }
+ }
+ }
+ }
+
+ private boolean isValidServiceReference(final String serviceId, final Class<? extends ControllerService> serviceClass) {
+ final Set<String> validServiceIds = controllerServiceProvider.getControllerServiceIdentifiers(serviceClass, getIdentifier());
+ return validServiceIds.contains(serviceId);
+ }
+
@Override
public void removeProcessor(final ProcessorNode processor) {
boolean removed = false;
@@ -2024,6 +2065,7 @@ public final class StandardProcessGroup implements ProcessGroup {
service.getVariableRegistry().setParent(getVariableRegistry());
this.controllerServices.put(service.getIdentifier(), service);
LOG.info("{} added to {}", service, this);
+ updateControllerServiceReferences(service);
onComponentModified();
} finally {
writeLock.unlock();
@@ -2751,8 +2793,8 @@ public final class StandardProcessGroup implements ProcessGroup {
}
}
- for (final String id : snippet.getProcessors().keySet()) {
- final ProcessorNode processorNode = getProcessor(id);
+ final Set<ProcessorNode> processors = findAllProcessors(snippet);
+ for (final ProcessorNode processorNode : processors) {
for (final PropertyDescriptor descriptor : processorNode.getProperties().keySet()) {
final Class<? extends ControllerService> serviceDefinition = descriptor.getControllerServiceDefinition();
@@ -2768,7 +2810,8 @@ public final class StandardProcessGroup implements ProcessGroup {
// ensure the configured service is an allowed service if it's still a valid service
if (currentControllerServiceIds.contains(serviceId) && !proposedControllerServiceIds.contains(serviceId)) {
- throw new IllegalStateException("Cannot perform Move Operation because a Processor references a service that is not available in the destination Process Group");
+ throw new IllegalStateException("Cannot perform Move Operation because Processor with ID " + processorNode.getIdentifier()
+ + " references a service that is not available in the destination Process Group");
}
}
}
@@ -2779,6 +2822,20 @@ public final class StandardProcessGroup implements ProcessGroup {
}
}
+ private Set<ProcessorNode> findAllProcessors(final Snippet snippet) {
+ final Set<ProcessorNode> processors = new HashSet<>();
+
+ snippet.getProcessors().keySet().stream()
+ .map(this::getProcessor)
+ .forEach(processors::add);
+
+ for (final String groupId : snippet.getProcessGroups().keySet()) {
+ processors.addAll(getProcessGroup(groupId).findAllProcessors());
+ }
+
+ return processors;
+ }
+
@Override
public MutableVariableRegistry getVariableRegistry() {
return variableRegistry;
http://git-wip-us.apache.org/repos/asf/nifi/blob/da94eaca/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java
index ed335e9..d45b51c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java
@@ -57,6 +57,8 @@ import org.apache.nifi.processor.StandardValidationContextFactory;
import org.apache.nifi.registry.VariableRegistry;
import org.apache.nifi.registry.variable.MutableVariableRegistry;
import org.apache.nifi.registry.variable.StandardComponentVariableRegistry;
+import org.apache.nifi.util.MockProcessContext;
+import org.apache.nifi.util.MockProcessorInitializationContext;
import org.apache.nifi.util.NiFiProperties;
import org.junit.Assert;
import org.junit.Before;
@@ -430,8 +432,14 @@ public class TestStandardControllerServiceProvider {
private ProcessorNode createProcessor(final StandardProcessScheduler scheduler, final ControllerServiceProvider serviceProvider) {
final ReloadComponent reloadComponent = Mockito.mock(ReloadComponent.class);
- final LoggableComponent<Processor> dummyProcessor = new LoggableComponent<>(new DummyProcessor(), systemBundle.getBundleDetails().getCoordinate(), null);
- final ProcessorNode procNode = new StandardProcessorNode(dummyProcessor, UUID.randomUUID().toString(),
+
+ final Processor processor = new DummyProcessor();
+ final MockProcessContext context = new MockProcessContext(processor, Mockito.mock(StateManager.class), variableRegistry);
+ final MockProcessorInitializationContext mockInitContext = new MockProcessorInitializationContext(processor, context);
+ processor.initialize(mockInitContext);
+
+ final LoggableComponent<Processor> dummyProcessor = new LoggableComponent<>(processor, systemBundle.getBundleDetails().getCoordinate(), null);
+ final ProcessorNode procNode = new StandardProcessorNode(dummyProcessor, mockInitContext.getIdentifier(),
new StandardValidationContextFactory(serviceProvider, null), scheduler, serviceProvider, niFiProperties,
new StandardComponentVariableRegistry(VariableRegistry.EMPTY_REGISTRY), reloadComponent);
[10/50] nifi git commit: NIFI-4441: patch avro maps in union types.
This closes #2207.
Posted by al...@apache.org.
NIFI-4441: patch avro maps in union types. This closes #2207.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6847c9e3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6847c9e3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6847c9e3
Branch: refs/heads/HDF-3.1-maint
Commit: 6847c9e3f528d56dbbf1ba5351e75fda40fa6d8a
Parents: ad0b160
Author: Patrice Freydiere <fr...@gmail.com>
Authored: Wed Oct 11 22:17:15 2017 +0200
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 14:59:53 2018 -0500
----------------------------------------------------------------------
.../nifi-avro-record-utils/pom.xml | 14 +++++++++++++
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 10 ++++++++++
.../org/apache/nifi/avro/TestAvroTypeUtil.java | 20 +++++++++++++++++++
.../resources/org/apache/nifi/avro/data.avro | Bin 0 -> 269 bytes
.../resources/org/apache/nifi/avro/schema.json | 20 +++++++++++++++++++
5 files changed, 64 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/6847c9e3/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
index afc04ef..5556088 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
@@ -42,4 +42,18 @@
<artifactId>nifi-record</artifactId>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/org/apache/nifi/avro/data.avro</exclude>
+ <exclude>src/test/resources/org/apache/nifi/avro/schema.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/6847c9e3/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index 5bdd70b..48f661f 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -626,6 +626,7 @@ public class AvroTypeUtil {
final Map<String, Object> values = new HashMap<>(recordSchema.getFieldCount());
for (final RecordField recordField : recordSchema.getFields()) {
+
Object value = avroRecord.get(recordField.getFieldName());
if (value == null) {
for (final String alias : recordField.getAliases()) {
@@ -637,6 +638,7 @@ public class AvroTypeUtil {
}
final String fieldName = recordField.getFieldName();
+ try {
final Field avroField = avroRecord.getSchema().getField(fieldName);
if (avroField == null) {
values.put(fieldName, null);
@@ -650,6 +652,10 @@ public class AvroTypeUtil {
final Object coercedValue = DataTypeUtils.convertType(rawValue, desiredType, fieldName);
values.put(fieldName, coercedValue);
+ } catch (Exception ex) {
+ logger.debug("fail to convert field " + fieldName, ex );
+ throw ex;
+ }
}
return values;
@@ -716,6 +722,10 @@ public class AvroTypeUtil {
return true;
}
break;
+ case MAP:
+ if (value instanceof Map) {
+ return true;
+ }
}
return DataTypeUtils.isCompatibleDataType(value, dataType);
http://git-wip-us.apache.org/repos/asf/nifi/blob/6847c9e3/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
index b017829..cf096aa 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
@@ -20,6 +20,7 @@ package org.apache.nifi.avro;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -28,6 +29,9 @@ import java.util.Optional;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.record.DataType;
@@ -239,4 +243,20 @@ public class TestAvroTypeUtil {
Assert.assertEquals(recordASchema, ((RecordDataType)recordBParentField.get().getDataType()).getChildSchema());
}
+ @Test
+ public void testMapWithNullSchema() throws IOException {
+
+ Schema recursiveSchema = new Schema.Parser().parse(getClass().getResourceAsStream("schema.json"));
+
+ // Make sure the following doesn't throw an exception
+ RecordSchema recordASchema = AvroTypeUtil.createSchema(recursiveSchema.getTypes().get(0));
+
+ // check the fix with the proper file
+ try(DataFileStream<GenericRecord> r = new DataFileStream<>(getClass().getResourceAsStream("data.avro"),
+ new GenericDatumReader<>())) {
+ GenericRecord n= r.next();
+ AvroTypeUtil.convertAvroRecordToMap(n, recordASchema);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6847c9e3/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro
new file mode 100644
index 0000000..139f244
Binary files /dev/null and b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/6847c9e3/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json
new file mode 100644
index 0000000..f6fd74b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json
@@ -0,0 +1,20 @@
+[ {
+ "namespace" : "net.a",
+ "type" : "record",
+ "name" : "O",
+ "fields" : [ {
+ "name" : "hash",
+ "type" : [ "null", {
+ "type" : "map",
+ "values" : "string"
+ } ]
+ } ]
+}, {
+ "namespace" : "net.a",
+ "type" : "record",
+ "name" : "A",
+ "fields" : [ {
+ "name" : "o",
+ "type" : [ "null", "O" ]
+ }]
+} ]
\ No newline at end of file
[08/50] nifi git commit: NIFI-4764: Add tooltips to status bar icons
add tooltip to process groups cleanup This closes #2394
Posted by al...@apache.org.
NIFI-4764: Add tooltips to status bar icons
add tooltip to process groups
cleanup
This closes #2394
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9ffdeb7c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9ffdeb7c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9ffdeb7c
Branch: refs/heads/HDF-3.1-maint
Commit: 9ffdeb7c8388724de879cbf9ba388e6f7286a53c
Parents: cadcd1d
Author: Marco Gaido <ma...@gmail.com>
Authored: Wed Jan 10 17:00:55 2018 +0100
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 14:57:18 2018 -0500
----------------------------------------------------------------------
nifi-docs/src/main/asciidoc/user-guide.adoc | 8 ++-
.../WEB-INF/partials/canvas/flow-status.jsp | 30 +++++-----
.../webapp/js/nf/canvas/nf-process-group.js | 60 +++++++++++++++-----
3 files changed, 69 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9ffdeb7c/nifi-docs/src/main/asciidoc/user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc
index e34ed2c..1a3877e 100644
--- a/nifi-docs/src/main/asciidoc/user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/user-guide.adoc
@@ -149,7 +149,13 @@ image::nifi-toolbar-components.png["NiFi Components Toolbar"]
The Components Toolbar runs across the top left portion of your screen. It consists of the components you can drag onto the
canvas to build your dataflow. Each component is described in more detail in link:building-dataflow.html[Building a Dataflow].
-The Status Bar is under the Components Toolbar. The Status bar provides information about the number of threads that are currently active in the flow, the amount of data that currently exists in the flow, how many Remote Process Groups exist on the canvas in each state (Transmitting, Not Transmitting), how many Processors exist on the canvas in each state (Stopped, Running, Invalid, Disabled), how many versioned Process Groups exist on the canvas in each state (Up to date, Locally modified, Stale, Locally modified and stale, Sync failure) and the timestamp at which all of this information was last refreshed. Additionally, if the instance of NiFi is clustered, the Status bar shows how many nodes are in the cluster and how many are currently connected.
+The Status Bar is under the Components Toolbar. The Status bar provides information about the number of threads that are
+currently active in the flow, the amount of data that currently exists in the flow, how many Remote Process Groups exist
+on the canvas in each state (Transmitting, Not Transmitting), how many Processors exist on the canvas in each state
+(Stopped, Running, Invalid, Disabled), how many versioned Process Groups exist on the canvas in each state (Up to date,
+Locally modified, Stale, Locally modified and stale, Sync failure) and the timestamp at which all of this information
+was last refreshed. Additionally, if the instance of NiFi is clustered, the Status bar shows how many nodes are in the
+cluster and how many are currently connected.
The Operate Palette sits to the left-hand side of the screen. It consists of buttons that are
used by DFMs to manage the flow, as well as by administrators who manage user access
http://git-wip-us.apache.org/repos/asf/nifi/blob/9ffdeb7c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/flow-status.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/flow-status.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/flow-status.jsp
index 9cb0ab9..5edfe10 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/flow-status.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/flow-status.jsp
@@ -17,23 +17,23 @@
<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
<div id="flow-status" flex layout="row" layout-align="space-between center">
<div id="flow-status-container" layout="row" layout-align="space-around center">
- <div class="fa fa-cubes" ng-if="appCtrl.nf.ClusterSummary.isClustered()"><span id="connected-nodes-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.connectedNodesCount}}</span></div>
- <div class="icon icon-threads"><span id="active-thread-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.activeThreadCount}}</span></div>
- <div class="fa fa-list"><span id="total-queued">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.totalQueued}}</span></div>
- <div class="fa fa-bullseye"><span id="controller-transmitting-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerTransmittingCount}}</span></div>
- <div class="icon icon-transmit-false"><span id="controller-not-transmitting-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerNotTransmittingCount}}</span></div>
- <div class="fa fa-play"><span id="controller-running-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerRunningCount}}</span></div>
- <div class="fa fa-stop"><span id="controller-stopped-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerStoppedCount}}</span></div>
- <div class="fa fa-warning"><span id="controller-invalid-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerInvalidCount}}</span></div>
- <div class="icon icon-enable-false"><span id="controller-disabled-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerDisabledCount}}</span></div>
- <div class="fa fa-check"><span id="controller-up-to-date-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerUpToDateCount}}</span></div>
- <div class="fa fa-asterisk"><span id="controller-locally-modified-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerLocallyModifiedCount}}</span></div>
- <div class="fa fa-arrow-circle-up"><span id="controller-stale-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerStaleCount}}</span></div>
- <div class="fa fa-exclamation-circle">
+ <div class="fa fa-cubes" ng-if="appCtrl.nf.ClusterSummary.isClustered()" title="Connected nodes / Total number of nodes in the cluster"><span id="connected-nodes-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.connectedNodesCount}}</span></div>
+ <div class="icon icon-threads" title="Active threads"><span id="active-thread-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.activeThreadCount}}</span></div>
+ <div class="fa fa-list" title="Total queued data"><span id="total-queued">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.totalQueued}}</span></div>
+ <div class="fa fa-bullseye" title="Transmitting Remote Process Groups"><span id="controller-transmitting-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerTransmittingCount}}</span></div>
+ <div class="icon icon-transmit-false" title="Not Transmitting Remote Process Groups"><span id="controller-not-transmitting-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerNotTransmittingCount}}</span></div>
+ <div class="fa fa-play" title="Running Components"><span id="controller-running-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerRunningCount}}</span></div>
+ <div class="fa fa-stop" title="Stopped Components"><span id="controller-stopped-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerStoppedCount}}</span></div>
+ <div class="fa fa-warning" title="Invalid Components"><span id="controller-invalid-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerInvalidCount}}</span></div>
+ <div class="icon icon-enable-false" title="Disabled Components"><span id="controller-disabled-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerDisabledCount}}</span></div>
+ <div class="fa fa-check" title="Up to date Versioned Process Groups"><span id="controller-up-to-date-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerUpToDateCount}}</span></div>
+ <div class="fa fa-asterisk" title="Locally modified Versioned Process Groups"><span id="controller-locally-modified-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerLocallyModifiedCount}}</span></div>
+ <div class="fa fa-arrow-circle-up" title="Stale Versioned Process Groups"><span id="controller-stale-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerStaleCount}}</span></div>
+ <div class="fa fa-exclamation-circle" title="Locally modified and stale Versioned Process Groups">
<span id="controller-locally-modified-and-stale-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerLocallyModifiedAndStaleCount}}</span>
</div>
- <div class="fa fa-question"><span id="controller-sync-failure-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerSyncFailureCount}}</span></div>
- <div class="fa fa-refresh"><span id="stats-last-refreshed">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.statsLastRefreshed}}</span></div>
+ <div class="fa fa-question" title="Sync failure Versioned Process Groups"><span id="controller-sync-failure-count">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.controllerSyncFailureCount}}</span></div>
+ <div class="fa fa-refresh" title="Last refresh"><span id="stats-last-refreshed">{{appCtrl.serviceProvider.headerCtrl.flowStatusCtrl.statsLastRefreshed}}</span></div>
<div id="canvas-loading-container" class="loading-container"></div>
</div>
<div layout="row" layout-align="end center">
http://git-wip-us.apache.org/repos/asf/nifi/blob/9ffdeb7c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-process-group.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-process-group.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-process-group.js
index 50a3818..dc7034a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-process-group.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-process-group.js
@@ -308,7 +308,10 @@
'class': 'process-group-transmitting process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf140');
+ .text('\uf140')
+ .append("title")
+ .text("Transmitting Remote Process Groups");
+
// transmitting count
details.append('text')
@@ -324,7 +327,9 @@
'class': 'process-group-not-transmitting process-group-contents-icon',
'font-family': 'flowfont'
})
- .text('\ue80a');
+ .text('\ue80a')
+ .append("title")
+ .text("Not Transmitting Remote Process Groups");
// not transmitting count
details.append('text')
@@ -340,7 +345,9 @@
'class': 'process-group-running process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf04b');
+ .text('\uf04b')
+ .append("title")
+ .text("Running Components");
// running count
details.append('text')
@@ -356,7 +363,9 @@
'class': 'process-group-stopped process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf04d');
+ .text('\uf04d')
+ .append("title")
+ .text("Stopped Components");
// stopped count
details.append('text')
@@ -372,7 +381,9 @@
'class': 'process-group-invalid process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf071');
+ .text('\uf071')
+ .append("title")
+ .text("Invalid Components");
// invalid count
details.append('text')
@@ -388,7 +399,9 @@
'class': 'process-group-disabled process-group-contents-icon',
'font-family': 'flowfont'
})
- .text('\ue802');
+ .text('\ue802')
+ .append("title")
+ .text("Disabled Components");
// disabled count
details.append('text')
@@ -407,7 +420,9 @@
'class': 'process-group-up-to-date process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf00c');
+ .text('\uf00c')
+ .append("title")
+ .text("Up to date Versioned Process Groups");
// up to date count
details.append('text')
@@ -427,7 +442,9 @@
'class': 'process-group-locally-modified process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf069');
+ .text('\uf069')
+ .append("title")
+ .text("Locally modified Versioned Process Groups");
// locally modified count
details.append('text')
@@ -447,7 +464,9 @@
'class': 'process-group-stale process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf0aa');
+ .text('\uf0aa')
+ .append("title")
+ .text("Stale Versioned Process Groups");
// stale count
details.append('text')
@@ -467,7 +486,9 @@
'class': 'process-group-locally-modified-and-stale process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf06a');
+ .text('\uf06a')
+ .append("title")
+ .text("Locally modified and stale Versioned Process Groups");
// locally modified and stale count
details.append('text')
@@ -487,7 +508,9 @@
'class': 'process-group-sync-failure process-group-contents-icon',
'font-family': 'FontAwesome'
})
- .text('\uf128');
+ .text('\uf128')
+ .append("title")
+ .text("Sync failure Versioned Process Groups");
// sync failure count
details.append('text')
@@ -847,6 +870,7 @@
.text(function (d) {
return d.activeRemotePortCount;
});
+ transmittingCount.append("title").text("Transmitting Remote Process Groups");
// update not transmitting
var notTransmitting = details.select('text.process-group-not-transmitting')
@@ -868,6 +892,7 @@
.text(function (d) {
return d.inactiveRemotePortCount;
});
+ notTransmittingCount.append("title").text("Not transmitting Remote Process Groups")
// update running
var running = details.select('text.process-group-running')
@@ -889,6 +914,7 @@
.text(function (d) {
return d.runningCount;
});
+ runningCount.append("title").text("Running Components");
// update stopped
var stopped = details.select('text.process-group-stopped')
@@ -910,6 +936,7 @@
.text(function (d) {
return d.stoppedCount;
});
+ stoppedCount.append("title").text("Stopped Components");
// update invalid
var invalid = details.select('text.process-group-invalid')
@@ -931,6 +958,7 @@
.text(function (d) {
return d.invalidCount;
});
+ invalidCount.append("title").text("Invalid Components");
// update disabled
var disabled = details.select('text.process-group-disabled')
@@ -944,7 +972,7 @@
var invalidX = parseInt(invalidCount.attr('x'), 10);
return invalidX + Math.round(invalidCount.node().getComputedTextLength()) + CONTENTS_SPACER;
});
- details.select('text.process-group-disabled-count')
+ var disabledCount = details.select('text.process-group-disabled-count')
.attr('x', function () {
var disabledCountX = parseInt(disabled.attr('x'), 10);
return disabledCountX + Math.round(disabled.node().getComputedTextLength()) + CONTENTS_VALUE_SPACER;
@@ -952,6 +980,7 @@
.text(function (d) {
return d.disabledCount;
});
+ disabledCount.append("title").text("Disabled Components");
// up to date current
var upToDate = details.select('text.process-group-up-to-date')
@@ -969,6 +998,7 @@
.text(function (d) {
return d.upToDateCount;
});
+ upToDateCount.append("title").text("Up to date Versioned Process Groups");
// update locally modified
var locallyModified = details.select('text.process-group-locally-modified')
@@ -990,6 +1020,7 @@
.text(function (d) {
return d.locallyModifiedCount;
});
+ locallyModifiedCount.append("title").text("Locally modified Versioned Process Groups");
// update stale
var stale = details.select('text.process-group-stale')
@@ -1011,6 +1042,7 @@
.text(function (d) {
return d.staleCount;
});
+ staleCount.append("title").text("Stale Versioned Process Groups");
// update locally modified and stale
var locallyModifiedAndStale = details.select('text.process-group-locally-modified-and-stale')
@@ -1032,6 +1064,7 @@
.text(function (d) {
return d.locallyModifiedAndStaleCount;
});
+ locallyModifiedAndStaleCount.append("title").text("Locally modified and stale Versioned Process Groups");
// update sync failure
var syncFailure = details.select('text.process-group-sync-failure')
@@ -1045,7 +1078,7 @@
var syncFailureX = parseInt(locallyModifiedAndStaleCount.attr('x'), 10);
return syncFailureX + Math.round(locallyModifiedAndStaleCount.node().getComputedTextLength()) + CONTENTS_SPACER - 2;
});
- details.select('text.process-group-sync-failure-count')
+ var syncFailureCount = details.select('text.process-group-sync-failure-count')
.attr('x', function () {
var syncFailureCountX = parseInt(syncFailure.attr('x'), 10);
return syncFailureCountX + Math.round(syncFailure.node().getComputedTextLength()) + CONTENTS_VALUE_SPACER;
@@ -1053,6 +1086,7 @@
.text(function (d) {
return d.syncFailureCount;
});
+ syncFailureCount.append("title").text("Sync failure Versioned Process Groups");
// update version control information
var versionControl = processGroup.select('text.version-control')
[06/50] nifi git commit: NIFI-4752: Addressed issue with some event
types having potentially the wrong FlowFile UUID listed (could have child
UUID when it's supposed to have parent flowfile UUID). In testing fix,
also found an issue with Search threads n
Posted by al...@apache.org.
NIFI-4752: Addressed issue with some event types having potentially the wrong FlowFile UUID listed (could have child UUID when it's supposed to have parent flowfile UUID). In testing fix, also found an issue with Search threads not being daemon and Re-Index threads not propertly being shutdown so addressed those as well.
This closes #2390.
Signed-off-by: Koji Kawamura <ij...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/658ce505
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/658ce505
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/658ce505
Branch: refs/heads/HDF-3.1-maint
Commit: 658ce505547efbee5ebb9f846077d3925a462719
Parents: 445f496
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Jan 9 13:34:41 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 14:53:59 2018 -0500
----------------------------------------------------------------------
.../provenance/lucene/SimpleIndexManager.java | 2 +-
.../schema/LookupTableEventRecord.java | 39 ++++++++++++++++----
.../store/WriteAheadStorePartition.java | 2 +
3 files changed, 35 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/658ce505/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
index 4d6c11d..d59a81d 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/SimpleIndexManager.java
@@ -53,7 +53,7 @@ public class SimpleIndexManager implements IndexManager {
public SimpleIndexManager(final RepositoryConfiguration repoConfig) {
this.repoConfig = repoConfig;
- this.searchExecutor = Executors.newFixedThreadPool(repoConfig.getQueryThreadPoolSize(), new NamedThreadFactory("Search Lucene Index"));
+ this.searchExecutor = Executors.newFixedThreadPool(repoConfig.getQueryThreadPoolSize(), new NamedThreadFactory("Search Lucene Index", true));
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/658ce505/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecord.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecord.java
index eccff2a..2fe1676 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecord.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecord.java
@@ -245,11 +245,14 @@ public class LookupTableEventRecord implements Record {
final Map<String, String> previousAttributes = truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.PREVIOUS_ATTRIBUTES), maxAttributeLength);
final Map<String, String> updatedAttributes = truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.UPDATED_ATTRIBUTES), maxAttributeLength);
+ final List<String> childUuids = (List<String>) record.getFieldValue(EventFieldNames.CHILD_UUIDS);
+ final List<String> parentUuids = (List<String>) record.getFieldValue(EventFieldNames.PARENT_UUIDS);
+
final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder();
builder.setAlternateIdentifierUri((String) record.getFieldValue(EventFieldNames.ALTERNATE_IDENTIFIER));
- builder.setChildUuids((List<String>) record.getFieldValue(EventFieldNames.CHILD_UUIDS));
+ builder.setChildUuids(childUuids);
builder.setDetails((String) record.getFieldValue(EventFieldNames.EVENT_DETAILS));
- builder.setParentUuids((List<String>) record.getFieldValue(EventFieldNames.PARENT_UUIDS));
+ builder.setParentUuids(parentUuids);
builder.setPreviousAttributes(previousAttributes);
builder.setRelationship((String) record.getFieldValue(EventFieldNames.RELATIONSHIP));
builder.setSourceSystemFlowFileIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_SYSTEM_FLOWFILE_IDENTIFIER));
@@ -263,20 +266,42 @@ public class LookupTableEventRecord implements Record {
// Determine the event type
final Integer eventTypeOrdinal = (Integer) record.getFieldValue(EventFieldNames.EVENT_TYPE);
+ ProvenanceEventType eventType;
if (eventTypeOrdinal == null || eventTypeOrdinal > eventTypes.size() || eventTypeOrdinal < 0) {
- builder.setEventType(ProvenanceEventType.UNKNOWN);
+ eventType = ProvenanceEventType.UNKNOWN;
} else {
try {
- builder.setEventType(ProvenanceEventType.valueOf(eventTypes.get(eventTypeOrdinal)));
+ eventType = ProvenanceEventType.valueOf(eventTypes.get(eventTypeOrdinal));
} catch (final Exception e) {
- builder.setEventType(ProvenanceEventType.UNKNOWN);
+ eventType = ProvenanceEventType.UNKNOWN;
}
}
+ builder.setEventType(eventType);
+
+ // Determine appropriate UUID for the event
+ String uuid = null;
+ switch (eventType) {
+ case CLONE:
+ case FORK:
+ case REPLAY:
+ if (parentUuids != null && !parentUuids.isEmpty()) {
+ uuid = parentUuids.get(0);
+ }
+ break;
+ case JOIN:
+ if (childUuids != null && !childUuids.isEmpty()) {
+ uuid = childUuids.get(0);
+ }
+ break;
+ }
- String uuid = updatedAttributes == null ? null : updatedAttributes.get(CoreAttributes.UUID.key());
if (uuid == null) {
- uuid = previousAttributes == null ? null : previousAttributes.get(CoreAttributes.UUID.key());
+ uuid = updatedAttributes == null ? null : updatedAttributes.get(CoreAttributes.UUID.key());
+ if (uuid == null) {
+ uuid = previousAttributes == null ? null : previousAttributes.get(CoreAttributes.UUID.key());
+ }
}
+
builder.setFlowFileUUID(uuid);
builder.setEventDuration((Integer) record.getFieldValue(EventFieldNames.EVENT_DURATION));
http://git-wip-us.apache.org/repos/asf/nifi/blob/658ce505/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
index 6c5cc8d..22d2a5f 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java
@@ -632,6 +632,8 @@ public class WriteAheadStorePartition implements EventStorePartition {
logger.error("Failed to re-index Provenance Events for partition " + partitionName, e);
}
+ executor.shutdown();
+
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
final long seconds = millis / 1000L;
final long millisRemainder = millis % 1000L;
[44/50] nifi git commit: NIFI-5045: Fixed error code handling in
PutHiveQL. This closes #2608
Posted by al...@apache.org.
NIFI-5045: Fixed error code handling in PutHiveQL. This closes #2608
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/8f988c3b
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/8f988c3b
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/8f988c3b
Branch: refs/heads/HDF-3.1-maint
Commit: 8f988c3b8a11f643e74580d08716d88090f6d143
Parents: 8a97e3e
Author: Matthew Burgess <ma...@apache.org>
Authored: Thu Apr 5 12:28:38 2018 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Apr 13 08:20:56 2018 -0400
----------------------------------------------------------------------
.../org/apache/nifi/processors/hive/PutHiveQL.java | 16 +++++++++++++++-
.../apache/nifi/processors/hive/TestPutHiveQL.java | 3 ++-
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/8f988c3b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java
index 93b07dc..c68bce8 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java
@@ -146,7 +146,21 @@ public class PutHiveQL extends AbstractHiveQLProcessor {
if (e instanceof SQLNonTransientException) {
return ErrorTypes.InvalidInput;
} else if (e instanceof SQLException) {
- return ErrorTypes.TemporalFailure;
+ // Use the SQLException's vendor code for guidance -- see Hive's ErrorMsg class for details on error codes
+ int errorCode = ((SQLException) e).getErrorCode();
+ if (errorCode >= 10000 && errorCode < 20000) {
+ return ErrorTypes.InvalidInput;
+ } else if (errorCode >= 20000 && errorCode < 30000) {
+ return ErrorTypes.TemporalFailure;
+ } else if (errorCode >= 30000 && errorCode < 40000) {
+ return ErrorTypes.TemporalInputFailure;
+ } else if (errorCode >= 40000 && errorCode < 50000) {
+ // These are unknown errors (to include some parse errors), but rather than generating an UnknownFailure which causes
+ // a ProcessException, we'll route to failure via an InvalidInput error type.
+ return ErrorTypes.InvalidInput;
+ } else {
+ return ErrorTypes.UnknownFailure;
+ }
} else {
return ErrorTypes.UnknownFailure;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8f988c3b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveQL.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveQL.java
index badc4d9..af737ae 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveQL.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHiveQL.java
@@ -773,7 +773,8 @@ public class TestPutHiveQL {
try {
if (++successful > allowedBeforeFailure) {
final Connection conn = Mockito.mock(Connection.class);
- Mockito.when(conn.prepareStatement(Mockito.any(String.class))).thenThrow(new SQLException("Unit Test Generated SQLException"));
+ // Throw a retryable error
+ Mockito.when(conn.prepareStatement(Mockito.any(String.class))).thenThrow(new SQLException("Unit Test Generated SQLException", "42000", 20000));
return conn;
} else {
return service.getConnection();
[04/50] nifi git commit: NIFI-4428: Changed versions in Druid bundle
for HDF 3.1.0.0
Posted by al...@apache.org.
NIFI-4428: Changed versions in Druid bundle for HDF 3.1.0.0
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/82092d66
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/82092d66
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/82092d66
Branch: refs/heads/HDF-3.1-maint
Commit: 82092d6669190d5eb99eb87be7f104dea555ef51
Parents: 45edd12
Author: Matthew Burgess <ma...@apache.org>
Authored: Tue Jan 16 14:10:35 2018 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Tue Jan 16 14:10:35 2018 -0500
----------------------------------------------------------------------
.../nifi-druid-controller-service-api-nar/pom.xml | 4 ++--
.../nifi-druid-controller-service-api/pom.xml | 2 +-
.../nifi-druid-controller-service/pom.xml | 6 +++---
nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml | 7 ++++---
.../nifi-druid-bundle/nifi-druid-processors/pom.xml | 10 +++++-----
nifi-nar-bundles/nifi-druid-bundle/pom.xml | 4 ++--
6 files changed, 17 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/82092d66/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
index bc400fb..4aa236c 100644
--- a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-bundle</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<artifactId>nifi-druid-controller-service-api-nar</artifactId>
@@ -34,7 +34,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-controller-service-api</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/82092d66/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
index 7883e3a..17d687b 100644
--- a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-bundle</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<artifactId>nifi-druid-controller-service-api</artifactId>
http://git-wip-us.apache.org/repos/asf/nifi/blob/82092d66/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
index d852fe1..c045a45 100644
--- a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-bundle</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<artifactId>nifi-druid-controller-service</artifactId>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-controller-service-api</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -49,7 +49,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/nifi/blob/82092d66/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
index 6a384c6..97c822d 100644
--- a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-bundle</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<artifactId>nifi-druid-nar</artifactId>
@@ -28,17 +28,18 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-controller-service-api-nar</artifactId>
+ <version>1.5.0</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-controller-service</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-processors</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/82092d66/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
index f596c6f..c65703f 100644
--- a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-bundle</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<artifactId>nifi-druid-processors</artifactId>
@@ -44,7 +44,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-controller-service-api</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -55,19 +55,19 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock-record-utils</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-druid-controller-service</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/nifi/blob/82092d66/nifi-nar-bundles/nifi-druid-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/pom.xml
index 82d3017..0aa9187 100644
--- a/nifi-nar-bundles/nifi-druid-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-druid-bundle/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<artifactId>nifi-druid-bundle</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<packaging>pom</packaging>
<properties>
[42/50] nifi git commit: NIFI-4631: Use java.nio.file.Files in
ListFile to improve performance This closes #2565.
Posted by al...@apache.org.
NIFI-4631: Use java.nio.file.Files in ListFile to improve performance
This closes #2565.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/5d53b457
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/5d53b457
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/5d53b457
Branch: refs/heads/HDF-3.1-maint
Commit: 5d53b457c02e7d98085d52e2344f0f00c2020c0c
Parents: ae6ad05
Author: Marco Gaido <ma...@gmail.com>
Authored: Mon Mar 19 14:48:27 2018 +0100
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Apr 4 08:44:10 2018 -0400
----------------------------------------------------------------------
.../nifi/processors/standard/ListFile.java | 112 ++++++++-----------
1 file changed, 49 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/5d53b457/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFile.java
index 5f2e2d2..4cf01b0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFile.java
@@ -40,7 +40,6 @@ import org.apache.nifi.processor.util.list.AbstractListProcessor;
import org.apache.nifi.processors.standard.util.FileInfo;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
@@ -64,7 +63,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiPredicate;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
@TriggerSerially
@InputRequirement(Requirement.INPUT_FORBIDDEN)
@@ -189,7 +191,7 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
private List<PropertyDescriptor> properties;
private Set<Relationship> relationships;
- private final AtomicReference<FileFilter> fileFilterRef = new AtomicReference<>();
+ private final AtomicReference<BiPredicate<Path, BasicFileAttributes>> fileFilterRef = new AtomicReference<BiPredicate<Path, BasicFileAttributes>>();
public static final String FILE_CREATION_TIME_ATTRIBUTE = "file.creationTime";
public static final String FILE_LAST_MODIFY_TIME_ATTRIBUTE = "file.lastModifiedTime";
@@ -311,9 +313,24 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
@Override
protected List<FileInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
- final File path = new File(getPath(context));
+ final Path path = new File(getPath(context)).toPath();
final Boolean recurse = context.getProperty(RECURSE).asBoolean();
- return scanDirectory(path, fileFilterRef.get(), recurse, minTimestamp);
+
+ final BiPredicate<Path, BasicFileAttributes> fileFilter = fileFilterRef.get();
+ int maxDepth = recurse? Integer.MAX_VALUE : 1;
+ Stream<Path> inputStream = Files.find(path, maxDepth, (p, attributes) ->
+ !attributes.isDirectory() && (minTimestamp == null || attributes.lastModifiedTime().toMillis() >= minTimestamp)
+ && fileFilter.test(p, attributes));
+ Stream<FileInfo> listing = inputStream.map(p -> {
+ File file = p.toFile();
+ return new FileInfo.Builder()
+ .directory(file.isDirectory())
+ .filename(file.getName())
+ .fullPathFileName(file.getAbsolutePath())
+ .lastModifiedTime(file.lastModified())
+ .build();
+ });
+ return listing.collect(Collectors.toList());
}
@Override
@@ -329,33 +346,7 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
|| IGNORE_HIDDEN_FILES.equals(property);
}
- private List<FileInfo> scanDirectory(final File path, final FileFilter filter, final Boolean recurse,
- final Long minTimestamp) throws IOException {
- final List<FileInfo> listing = new ArrayList<>();
- File[] files = path.listFiles();
- if (files != null) {
- for (File file : files) {
- if (file.isDirectory()) {
- if (recurse) {
- listing.addAll(scanDirectory(file, filter, true, minTimestamp));
- }
- } else {
- if ((minTimestamp == null || file.lastModified() >= minTimestamp) && filter.accept(file)) {
- listing.add(new FileInfo.Builder()
- .directory(file.isDirectory())
- .filename(file.getName())
- .fullPathFileName(file.getAbsolutePath())
- .lastModifiedTime(file.lastModified())
- .build());
- }
- }
- }
- }
-
- return listing;
- }
-
- private FileFilter createFileFilter(final ProcessContext context) {
+ private BiPredicate<Path, BasicFileAttributes> createFileFilter(final ProcessContext context) {
final long minSize = context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue();
final Double maxSize = context.getProperty(MAX_SIZE).asDataSize(DataUnit.B);
final long minAge = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
@@ -366,41 +357,36 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
final boolean recurseDirs = context.getProperty(RECURSE).asBoolean();
final String pathPatternStr = context.getProperty(PATH_FILTER).getValue();
final Pattern pathPattern = (!recurseDirs || pathPatternStr == null) ? null : Pattern.compile(pathPatternStr);
-
- return new FileFilter() {
- @Override
- public boolean accept(final File file) {
- if (minSize > file.length()) {
- return false;
- }
- if (maxSize != null && maxSize < file.length()) {
- return false;
- }
- final long fileAge = System.currentTimeMillis() - file.lastModified();
- if (minAge > fileAge) {
- return false;
- }
- if (maxAge != null && maxAge < fileAge) {
- return false;
- }
- if (ignoreHidden && file.isHidden()) {
- return false;
- }
- if (pathPattern != null) {
- Path reldir = Paths.get(indir).relativize(file.toPath()).getParent();
- if (reldir != null && !reldir.toString().isEmpty()) {
- if (!pathPattern.matcher(reldir.toString()).matches()) {
- return false;
- }
+ return (path, attributes) -> {
+ if (minSize > attributes.size()) {
+ return false;
+ }
+ if (maxSize != null && maxSize < attributes.size()) {
+ return false;
+ }
+ final long fileAge = System.currentTimeMillis() - attributes.lastModifiedTime().toMillis();
+ if (minAge > fileAge) {
+ return false;
+ }
+ if (maxAge != null && maxAge < fileAge) {
+ return false;
+ }
+ if (ignoreHidden && path.toFile().isHidden()) {
+ return false;
+ }
+ if (pathPattern != null) {
+ Path reldir = Paths.get(indir).relativize(path).getParent();
+ if (reldir != null && !reldir.toString().isEmpty()) {
+ if (!pathPattern.matcher(reldir.toString()).matches()) {
+ return false;
}
}
- //Verify that we have at least read permissions on the file we're considering grabbing
- if (!Files.isReadable(file.toPath())) {
- return false;
- }
- return filePattern.matcher(file.getName()).matches();
}
+ // Verify that we have at least read permissions on the file we're considering grabbing
+ if (!Files.isReadable(path)) {
+ return false;
+ }
+ return filePattern.matcher(path.getFileName().toString()).matches();
};
}
-
}
[13/50] nifi git commit: NIFI-4782: Allow the value of a Required
Property to be moved when changing version of a flow or reverting a flow
Posted by al...@apache.org.
NIFI-4782: Allow the value of a Required Property to be moved when changing version of a flow or reverting a flow
This closes #2406.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4d14368d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4d14368d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4d14368d
Branch: refs/heads/HDF-3.1-maint
Commit: 4d14368d1d3940b0e11d28f542e1ca2e6c1d861f
Parents: fbc4ec3
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Jan 16 12:36:52 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 15:03:46 2018 -0500
----------------------------------------------------------------------
.../nifi/controller/AbstractConfiguredComponent.java | 12 ++++++++----
.../org/apache/nifi/controller/ConfiguredComponent.java | 6 +++++-
.../org/apache/nifi/groups/StandardProcessGroup.java | 8 ++++++--
3 files changed, 19 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/4d14368d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractConfiguredComponent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractConfiguredComponent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractConfiguredComponent.java
index ff5a8af..a5f2631 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractConfiguredComponent.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractConfiguredComponent.java
@@ -151,7 +151,7 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
}
@Override
- public void setProperties(Map<String, String> properties) {
+ public void setProperties(final Map<String, String> properties, final boolean allowRemovalOfRequiredProperties) {
if (properties == null) {
return;
}
@@ -170,7 +170,7 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
}
if (entry.getKey() != null && entry.getValue() == null) {
- removeProperty(entry.getKey());
+ removeProperty(entry.getKey(), allowRemovalOfRequiredProperties);
} else if (entry.getKey() != null) {
setProperty(entry.getKey(), CharacterFilterUtils.filterInvalidXmlCharacters(entry.getValue()));
}
@@ -231,17 +231,20 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
* if was a dynamic property.
*
* @param name the property to remove
+ * @param allowRemovalOfRequiredProperties whether or not the property should be removed if it's required
* @return true if removed; false otherwise
* @throws java.lang.IllegalArgumentException if the name is null
*/
- private boolean removeProperty(final String name) {
+ private boolean removeProperty(final String name, final boolean allowRemovalOfRequiredProperties) {
if (null == name) {
throw new IllegalArgumentException("Name can not be null");
}
final PropertyDescriptor descriptor = getComponent().getPropertyDescriptor(name);
String value = null;
- if (!descriptor.isRequired() && (value = properties.remove(descriptor)) != null) {
+
+ final boolean allowRemoval = allowRemovalOfRequiredProperties || !descriptor.isRequired();
+ if (allowRemoval && (value = properties.remove(descriptor)) != null) {
if (descriptor.getControllerServiceDefinition() != null) {
if (value != null) {
@@ -541,6 +544,7 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
}
}
+ @Override
public ComponentVariableRegistry getVariableRegistry() {
return this.variableRegistry;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/4d14368d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ConfiguredComponent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ConfiguredComponent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ConfiguredComponent.java
index 940ac21..b0f65a7 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ConfiguredComponent.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ConfiguredComponent.java
@@ -50,7 +50,11 @@ public interface ConfiguredComponent extends ComponentAuthorizable {
public void setAnnotationData(String data);
- public void setProperties(Map<String, String> properties);
+ public default void setProperties(Map<String, String> properties) {
+ setProperties(properties, false);
+ }
+
+ public void setProperties(Map<String, String> properties, boolean allowRemovalOfRequiredProperties);
public Map<PropertyDescriptor, String> getProperties();
http://git-wip-us.apache.org/repos/asf/nifi/blob/4d14368d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index baf02bc..15f2b5f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -3925,7 +3925,9 @@ public final class StandardProcessGroup implements ProcessGroup {
service.setAnnotationData(proposed.getAnnotationData());
service.setComments(proposed.getComments());
service.setName(proposed.getName());
- service.setProperties(populatePropertiesMap(service.getProperties(), proposed.getProperties(), proposed.getPropertyDescriptors(), service.getProcessGroup()));
+
+ final Map<String, String> properties = populatePropertiesMap(service.getProperties(), proposed.getProperties(), proposed.getPropertyDescriptors(), service.getProcessGroup());
+ service.setProperties(properties, true);
if (!isEqual(service.getBundleCoordinate(), proposed.getBundle())) {
final BundleCoordinate newBundleCoordinate = toCoordinate(proposed.getBundle());
@@ -4045,7 +4047,9 @@ public final class StandardProcessGroup implements ProcessGroup {
processor.setExecutionNode(ExecutionNode.valueOf(proposed.getExecutionNode()));
processor.setName(proposed.getName());
processor.setPenalizationPeriod(proposed.getPenaltyDuration());
- processor.setProperties(populatePropertiesMap(processor.getProperties(), proposed.getProperties(), proposed.getPropertyDescriptors(), processor.getProcessGroup()));
+
+ final Map<String, String> properties = populatePropertiesMap(processor.getProperties(), proposed.getProperties(), proposed.getPropertyDescriptors(), processor.getProcessGroup());
+ processor.setProperties(properties, true);
processor.setRunDuration(proposed.getRunDurationMillis(), TimeUnit.MILLISECONDS);
processor.setScheduldingPeriod(proposed.getSchedulingPeriod());
processor.setSchedulingStrategy(SchedulingStrategy.valueOf(proposed.getSchedulingStrategy()));
[03/50] nifi git commit: NIFI-4428: Replaced JSON input with
RecordReader using per-record flowfiles NIFI-4428: Added initial L&N,
fixed dependency hierarchy
Posted by al...@apache.org.
NIFI-4428: Replaced JSON input with RecordReader
using per-record flowfiles
NIFI-4428: Added initial L&N, fixed dependency hierarchy
NIFI-4428: Exposed additional ZK properties
NIFI-4428: More work to reduce dependencies in Druid artifacts
NIFI-4428: Removed inaccessible attribute
NIFI-4428: Fixed typo in property description
NIFI-4428: Updates to Druid NAR L&N
NIFI-4428: Updated LICENSE to refer to ICU License for icu4j
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/45edd12a
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/45edd12a
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/45edd12a
Branch: refs/heads/HDF-3.1-maint
Commit: 45edd12a01628a9b6424d41457753fe16c6894a2
Parents: 46d30c7
Author: Matthew Burgess <ma...@apache.org>
Authored: Wed Nov 22 11:36:30 2017 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Tue Jan 16 13:46:16 2018 -0500
----------------------------------------------------------------------
nifi-assembly/LICENSE | 90 +++
nifi-assembly/NOTICE | 155 +++++
.../record/util/DataTypeUtils.java | 86 ++-
.../serialization/record/TestDataTypeUtils.java | 134 +++-
.../pom.xml | 40 ++
.../src/main/resources/META-INF/LICENSE | 329 ++++++++++
.../src/main/resources/META-INF/NOTICE | 612 ++++++++++++++++++
.../nifi-druid-controller-service-api/pom.xml | 137 +++++
.../api/druid/DruidTranquilityService.java | 30 +
.../nifi-druid-controller-service/pom.xml | 56 ++
.../druid/DruidTranquilityController.java | 614 +++++++++++++++++++
...org.apache.nifi.controller.ControllerService | 16 +
.../druid/DruidTranquilityControllerTest.java | 100 +++
.../nifi-druid-bundle/nifi-druid-nar/pom.xml | 44 ++
.../src/main/resources/META-INF/LICENSE | 209 +++++++
.../src/main/resources/META-INF/NOTICE | 5 +
.../nifi-druid-processors/pom.xml | 74 +++
.../nifi/processors/druid/PutDruidRecord.java | 373 +++++++++++
.../org.apache.nifi.processor.Processor | 16 +
.../druid/MockDruidTranquilityController.java | 150 +++++
.../processors/druid/PutDruidRecordTest.java | 113 ++++
nifi-nar-bundles/nifi-druid-bundle/pom.xml | 56 ++
nifi-nar-bundles/pom.xml | 1 +
23 files changed, 3434 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-assembly/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-assembly/LICENSE b/nifi-assembly/LICENSE
index 8760829..0edf519 100644
--- a/nifi-assembly/LICENSE
+++ b/nifi-assembly/LICENSE
@@ -2073,3 +2073,93 @@ style license.
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
+
+ The binary distribution of this product bundles 'ANTLR 4' which is available
+ under a "3-clause BSD" license. For details see http://www.antlr.org/license.html
+
+ Copyright (c) 2012 Terence Parr and Sam Harwell
+ All rights reserved.
+ Redistribution and use in source and binary forms, with or without modification, are permitted
+ provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this list of
+ conditions and the following disclaimer.
+ Redistributions in binary form must reproduce the above copyright notice, this list of
+ conditions and the following disclaimer in the documentation and/or other materials
+ provided with the distribution.
+
+ Neither the name of the author nor the names of its contributors may be used to endorse
+ or promote products derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'icu4j'
+ which is available under a X-style license.
+
+ COPYRIGHT AND PERMISSION NOTICE
+
+ Copyright (c) 1995-2011 International Business Machines Corporation and others
+
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+ and associated documentation files (the "Software"), to deal in the Software without restriction,
+ including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or
+ sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+ provided that the above copyright notice(s) and this permission notice appear in all copies of
+ the Software and that both the above copyright notice(s) and this permission notice appear in
+ supporting documentation.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED
+ IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY
+ DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+ NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ OF THIS SOFTWARE.
+
+ Except as contained in this notice, the name of a copyright holder shall not be used in advertising
+ or otherwise to promote the sale, use or other dealings in this Software without prior written
+ authorization of the copyright holder.
+
+ The binary distribution of this product bundles 'Scala Library, Scala Parser, Scala Reflect' which is available
+ under a "3-clause BSD" license. For details see https://github.com/scala/scala/blob/v2.11.7/doc/LICENSE.md)
+
+ This software includes projects with other licenses -- see `doc/LICENSE.md`.
+
+ Copyright (c) 2002-2017 EPFL
+ Copyright (c) 2011-2017 Lightbend, Inc.
+
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the EPFL nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-assembly/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE
index ac5456a..3f2a24c 100644
--- a/nifi-assembly/NOTICE
+++ b/nifi-assembly/NOTICE
@@ -1553,6 +1553,160 @@ The following binary components are provided under the Apache Software License v
The following NOTICE information applies:
DataStax Java Driver for Apache Cassandra - Core
Copyright (C) 2012-2017 DataStax Inc.
+ (ASLv2) bytebuffer-collections
+ The following NOTICE information applies:
+ bytebuffer-collections
+ Copyright 2011-2015 Metamarkets Group Inc.
+
+ (ASLv2) Druid
+ The following NOTICE information applies:
+ Druid - a distributed column store.
+ Copyright 2012-2016 Metamarkets Group Inc.
+ Copyright 2015-2016 Yahoo! Inc.
+ Copyright 2015-2016 Imply Data, Inc.
+
+ -------------------------------------------------------------------------------
+
+ This product contains a modified version of Andrew Duffy's java-alphanum library
+ * LICENSE:
+ * https://github.com/amjjd/java-alphanum/blob/5c036e2e492cc7f3b7bcdebd46b8f9e2a87927e5/LICENSE.txt (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/amjjd/java-alphanum
+
+ This product contains conjunctive normal form conversion code and a variance aggregator algorithm adapted from Apache Hive
+ * LICENSE:
+ * https://github.com/apache/hive/blob/branch-2.0/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/apache/hive
+
+ This product contains variable length long deserialization code adapted from Apache Lucene
+ * LICENSE:
+ * https://github.com/apache/lucene-solr/blob/master/lucene/LICENSE.txt (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/apache/lucene-solr
+
+ This product contains a modified version of Metamarkets java-util library
+ * LICENSE:
+ * https://github.com/metamx/java-util/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/metamx/java-util
+ * COMMIT TAG:
+ * https://github.com/metamx/java-util/commit/826021f
+
+ This product contains a modified version of TestNG 6.8.7
+ * LICENSE:
+ * http://testng.org/license/ (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * http://testng.org/
+
+ This product contains a modified version of Metamarkets bytebuffer-collections library
+ * LICENSE:
+ * https://github.com/metamx/bytebuffer-collections/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/metamx/bytebuffer-collections
+ * COMMIT TAG:
+ * https://github.com/metamx/bytebuffer-collections/commit/3d1e7c8
+
+ This product contains SQL query planning code adapted from Apache Calcite
+ * LICENSE:
+ * https://github.com/apache/calcite/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://calcite.apache.org/
+
+ This product contains a modified version of Metamarkets extendedset library
+ * LICENSE:
+ * https://github.com/metamx/extendedset/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/metamx/extendedset
+ * COMMIT TAG:
+ * https://github.com/metamx/extendedset/commit/c9d647d
+
+ This product contains a modified version of Alessandro Colantonio's CONCISE
+ (COmpressed 'N' Composable Integer SEt) library, extending the functionality of
+ ConciseSet to use IntBuffers.
+ * (c) 2010 Alessandro Colantonio
+ * <ma...@mat.uniroma3.it>
+ * <http://ricerca.mat.uniroma3.it/users/colanton>
+ * LICENSE:
+ * Apache License, Version 2.0
+ * HOMEPAGE:
+ * https://sourceforge.net/projects/concise/
+
+ This product contains a modified version of The Guava Authors's Closer class from Guava library:
+ * LICENSE:
+ * https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c/COPYING (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/google/guava
+ * COMMIT TAG:
+ * https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c
+
+ This product contains code adapted from Apache Hadoop
+ * LICENSE:
+ * https://github.com/apache/hadoop/blob/trunk/LICENSE.txt (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * http://hadoop.apache.org/
+
+ (ASLv2) Emitter
+ The following NOTICE information applies:
+ emitter
+ Copyright 2012-2015 Metamarkets Group Inc.
+
+ (ASLv2) Finagle
+ The following NOTICE information applies:
+ Copyright 2017, Twitter Inc.
+ This software contains portions of code from other open-source projects, including:
+ >>>>>>>>>>>>>>
+ Apache Thrift
+ Copyright 2006-2010 The Apache Software Foundation.
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ (ASLv2) java-util
+ The following NOTICE information applies:
+ java-util
+ Copyright 2011-2017 Metamarkets Group Inc.
+
+ (ASLv2) jdbi
+ The following NOTICE information applies:
+ This product includes software developed by Brian McCallister
+
+ (ASLv2) Joda Convert
+ The following NOTICE information applies:
+ Joda Convert
+ Copyright 2010-present Stephen Colebourne
+
+ This product includes software developed by
+ Joda.org (http://www.joda.org/).
+
+
+ Joda-Convert includes code from Google Guava, which is licensed as follows:
+
+ Copyright (C) 2011 The Guava Authors
+
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+
+ (ASLv2) loglady
+ The following NOTICE information applies:
+ Copyright (c) 2013 Daniel Lundin
+
+ (ASLv2) scala_util
+ The following NOTICE information applies:
+ Copyright 2012 Metamarkets Group Inc.
+
+ (ASLv2) server-metrics
+ The following NOTICE information applies:
+ server-metrics
+ Copyright 2011-2015 Metamarkets Group Inc.
+
************************
Common Development and Distribution License 1.1
@@ -1663,6 +1817,7 @@ Mozilla Public License v2.0
The following binary components are provided under the Mozilla Public License v2.0. See project link for details.
(MPL 2.0) Saxon HE (net.sf.saxon:Saxon-HE:jar:9.6.0-5 - http://www.saxonica.com/)
+ (MPL 2.0) Rhino (org.mozilla:rhino:jar:1.7R5 - https://developer.mozilla.org/en-US/docs/Mozilla/Projects/Rhino/Download_Rhino)
*****************
Mozilla Public License v1.1
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
index 6063d3b..145c17c 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -45,12 +46,14 @@ import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.type.ArrayDataType;
import org.apache.nifi.serialization.record.type.ChoiceDataType;
+import org.apache.nifi.serialization.record.type.MapDataType;
import org.apache.nifi.serialization.record.type.RecordDataType;
public class DataTypeUtils {
- // Regexes for parsing Floting-Point numbers
+ // Regexes for parsing Floating-Point numbers
private static final String OptionalSign = "[\\-\\+]?";
private static final String Infinity = "(Infinity)";
private static final String NotANumber = "(NaN)";
@@ -326,6 +329,87 @@ public class DataTypeUtils {
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Map for field " + fieldName);
}
+ /**
+ * Creates a native Java object from a given object of a specified type. Non-scalar (complex, nested, etc.) data types are processed iteratively/recursively, such that all
+ * included objects are native Java objects, rather than Record API objects or implementation-specific objects.
+ * @param value The object to be converted
+ * @param dataType The type of the provided object
+ * @return An object representing a native Java conversion of the given input object
+ */
+ public static Object convertRecordFieldtoObject(final Object value, final DataType dataType) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof Record) {
+ Record record = (Record) value;
+ RecordSchema recordSchema = record.getSchema();
+ if (recordSchema == null) {
+ throw new IllegalTypeConversionException("Cannot convert value of type Record to Map because Record does not have an associated Schema");
+ }
+ final Map<String, Object> recordMap = new HashMap<>();
+ for (RecordField field : recordSchema.getFields()) {
+ final DataType fieldDataType = field.getDataType();
+ final String fieldName = field.getFieldName();
+ Object fieldValue = record.getValue(fieldName);
+ if (fieldValue == null) {
+ recordMap.put(fieldName, null);
+ } else if (isScalarValue(fieldDataType, fieldValue)) {
+ recordMap.put(fieldName, fieldValue);
+
+ } else if (fieldDataType instanceof RecordDataType) {
+ Record nestedRecord = (Record) fieldValue;
+ recordMap.put(fieldName, convertRecordFieldtoObject(nestedRecord, fieldDataType));
+
+ } else if (fieldDataType instanceof MapDataType) {
+ recordMap.put(fieldName, convertRecordMapToJavaMap((Map) fieldValue, ((MapDataType)fieldDataType).getValueType()));
+
+ } else if (fieldDataType instanceof ArrayDataType) {
+ recordMap.put(fieldName, convertRecordArrayToJavaArray((Object[])fieldValue, ((ArrayDataType) fieldDataType).getElementType()));
+ } else {
+ throw new IllegalTypeConversionException("Cannot convert value [" + fieldValue + "] of type " + fieldDataType.toString()
+ + " to Map for field " + fieldName + " because the type is not supported");
+ }
+ }
+ return recordMap;
+ } else if (value instanceof Map) {
+ return convertRecordMapToJavaMap((Map)value, ((MapDataType)dataType).getValueType());
+ } else if (dataType != null && isScalarValue(dataType, value)) {
+ return value;
+ }
+
+ throw new IllegalTypeConversionException("Cannot convert value of class " + value.getClass().getName() + " because the type is not supported");
+ }
+
+
+ public static Map<String, Object> convertRecordMapToJavaMap(final Map<String, Object> map, DataType valueDataType) {
+
+ if (map == null) {
+ return null;
+ }
+
+ Map<String, Object> resultMap = new LinkedHashMap<>();
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ resultMap.put(entry.getKey(), convertRecordFieldtoObject(entry.getValue(), valueDataType));
+ }
+ return resultMap;
+ }
+
+ public static Object[] convertRecordArrayToJavaArray(final Object[] array, DataType elementDataType) {
+
+ if (array == null || array.length == 0 || isScalarValue(elementDataType, array[0])) {
+ return array;
+ } else {
+ // Must be an array of complex types, build an array of converted values
+ Object[] resultArray = new Object[array.length];
+ for (int i = 0; i < array.length; i++) {
+ resultArray[i] = convertRecordFieldtoObject(array[i], elementDataType);
+ }
+ return resultArray;
+ }
+ }
+
public static boolean isMapTypeCompatible(final Object value) {
return value != null && value instanceof Map;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
index 6666651..a239ea7 100644
--- a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
@@ -17,11 +17,21 @@
package org.apache.nifi.serialization.record;
+import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.record.util.DataTypeUtils;
-import org.junit.Assert;
import org.junit.Test;
import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
public class TestDataTypeUtils {
/**
@@ -33,12 +43,126 @@ public class TestDataTypeUtils {
java.util.Date date = new java.util.Date();
Timestamp ts = DataTypeUtils.toTimestamp(date, null, null);
- Assert.assertNotNull(ts);
- Assert.assertEquals("Times didn't match", ts.getTime(), date.getTime());
+ assertNotNull(ts);
+ assertEquals("Times didn't match", ts.getTime(), date.getTime());
java.sql.Date sDate = new java.sql.Date(date.getTime());
ts = DataTypeUtils.toTimestamp(date, null, null);
- Assert.assertNotNull(ts);
- Assert.assertEquals("Times didn't match", ts.getTime(), sDate.getTime());
+ assertNotNull(ts);
+ assertEquals("Times didn't match", ts.getTime(), sDate.getTime());
+ }
+
+ @Test
+ public void testConvertRecordMapToJavaMap() {
+ assertNull(DataTypeUtils.convertRecordMapToJavaMap(null, null));
+ assertNull(DataTypeUtils.convertRecordMapToJavaMap(null, RecordFieldType.MAP.getDataType()));
+ Map<String,Object> resultMap = DataTypeUtils.convertRecordMapToJavaMap(new HashMap<>(), RecordFieldType.MAP.getDataType());
+ assertNotNull(resultMap);
+ assertTrue(resultMap.isEmpty());
+
+ int[] intArray = {3,2,1};
+
+ Map<String,Object> inputMap = new HashMap<String,Object>() {{
+ put("field1", "hello");
+ put("field2", 1);
+ put("field3", intArray);
+ }};
+
+ resultMap = DataTypeUtils.convertRecordMapToJavaMap(inputMap, RecordFieldType.STRING.getDataType());
+ assertNotNull(resultMap);
+ assertFalse(resultMap.isEmpty());
+ assertEquals("hello", resultMap.get("field1"));
+ assertEquals(1, resultMap.get("field2"));
+ assertTrue(resultMap.get("field3") instanceof int[]);
+ assertNull(resultMap.get("field4"));
+
+ }
+
+ @Test
+ public void testConvertRecordArrayToJavaArray() {
+ assertNull(DataTypeUtils.convertRecordArrayToJavaArray(null, null));
+ assertNull(DataTypeUtils.convertRecordArrayToJavaArray(null, RecordFieldType.STRING.getDataType()));
+ String[] stringArray = {"Hello", "World!"};
+ Object[] resultArray = DataTypeUtils.convertRecordArrayToJavaArray(stringArray, RecordFieldType.STRING.getDataType());
+ assertNotNull(resultArray);
+ for(Object o : resultArray) {
+ assertTrue(o instanceof String);
+ }
+ }
+
+ @Test
+ public void testConvertRecordFieldToObject() {
+ assertNull(DataTypeUtils.convertRecordFieldtoObject(null, null));
+ assertNull(DataTypeUtils.convertRecordFieldtoObject(null, RecordFieldType.MAP.getDataType()));
+
+ final List<RecordField> fields = new ArrayList<>();
+ fields.add(new RecordField("defaultOfHello", RecordFieldType.STRING.getDataType(), "hello"));
+ fields.add(new RecordField("noDefault", RecordFieldType.CHOICE.getChoiceDataType(RecordFieldType.STRING.getDataType())));
+ fields.add(new RecordField("intField", RecordFieldType.INT.getDataType()));
+ fields.add(new RecordField("intArray", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType())));
+
+ // Map of Records with Arrays
+ List<RecordField> nestedRecordFields = new ArrayList<>();
+ nestedRecordFields.add(new RecordField("a", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType())));
+ nestedRecordFields.add(new RecordField("b", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType())));
+ RecordSchema nestedRecordSchema = new SimpleRecordSchema(nestedRecordFields);
+
+ fields.add(new RecordField("complex", RecordFieldType.MAP.getMapDataType(RecordFieldType.RECORD.getRecordDataType(nestedRecordSchema))));
+
+ final RecordSchema schema = new SimpleRecordSchema(fields);
+ final Map<String, Object> values = new HashMap<>();
+ values.put("noDefault", "world");
+ values.put("intField", 5);
+ values.put("intArray", new Integer[] {3,2,1});
+ final Map<String, Object> complexValues = new HashMap<>();
+
+ final Map<String, Object> complexValueRecord1 = new HashMap<>();
+ complexValueRecord1.put("a",new Integer[] {3,2,1});
+ complexValueRecord1.put("b",new Integer[] {5,4,3});
+
+ final Map<String, Object> complexValueRecord2 = new HashMap<>();
+ complexValueRecord2.put("a",new String[] {"hello","world!"});
+ complexValueRecord2.put("b",new String[] {"5","4","3"});
+
+ complexValues.put("complex1", DataTypeUtils.toRecord(complexValueRecord1, nestedRecordSchema, "complex1"));
+ complexValues.put("complex2", DataTypeUtils.toRecord(complexValueRecord2, nestedRecordSchema, "complex2"));
+
+ values.put("complex", complexValues);
+ final Record inputRecord = new MapRecord(schema, values);
+
+ Object o = DataTypeUtils.convertRecordFieldtoObject(inputRecord, RecordFieldType.RECORD.getRecordDataType(schema));
+ assertTrue(o instanceof Map);
+ Map<String,Object> outputMap = (Map<String,Object>) o;
+ assertEquals("hello", outputMap.get("defaultOfHello"));
+ assertEquals("world", outputMap.get("noDefault"));
+ o = outputMap.get("intField");
+ assertEquals(5,o);
+ o = outputMap.get("intArray");
+ assertTrue(o instanceof Integer[]);
+ Integer[] intArray = (Integer[])o;
+ assertEquals(3, intArray.length);
+ assertEquals((Integer)3, intArray[0]);
+ o = outputMap.get("complex");
+ assertTrue(o instanceof Map);
+ Map<String,Object> nestedOutputMap = (Map<String,Object>)o;
+ o = nestedOutputMap.get("complex1");
+ assertTrue(o instanceof Map);
+ Map<String,Object> complex1 = (Map<String,Object>)o;
+ o = complex1.get("a");
+ assertTrue(o instanceof Integer[]);
+ assertEquals((Integer)2, ((Integer[])o)[1]);
+ o = complex1.get("b");
+ assertTrue(o instanceof Integer[]);
+ assertEquals((Integer)3, ((Integer[])o)[2]);
+ o = nestedOutputMap.get("complex2");
+ assertTrue(o instanceof Map);
+ Map<String,Object> complex2 = (Map<String,Object>)o;
+ o = complex2.get("a");
+ assertTrue(o instanceof String[]);
+ assertEquals("hello", ((String[])o)[0]);
+ o = complex2.get("b");
+ assertTrue(o instanceof String[]);
+ assertEquals("4", ((String[])o)[1]);
+
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
new file mode 100644
index 0000000..bc400fb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/pom.xml
@@ -0,0 +1,40 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-bundle</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-druid-controller-service-api-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service-api</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000..9efc253
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,329 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+The binary distribution of this product bundles 'ANTLR 4' which is available
+ under a "3-clause BSD" license. For details see http://www.antlr.org/license.html
+
+ Copyright (c) 2012 Terence Parr and Sam Harwell
+ All rights reserved.
+ Redistribution and use in source and binary forms, with or without modification, are permitted
+ provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this list of
+ conditions and the following disclaimer.
+ Redistributions in binary form must reproduce the above copyright notice, this list of
+ conditions and the following disclaimer in the documentation and/or other materials
+ provided with the distribution.
+
+ Neither the name of the author nor the names of its contributors may be used to endorse
+ or promote products derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'ParaNamer'
+ which is available under a BSD style license.
+
+ Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'icu4j'
+ which is available under the ICU License.
+
+ COPYRIGHT AND PERMISSION NOTICE
+
+ Copyright (c) 1995-2011 International Business Machines Corporation and others
+
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+ and associated documentation files (the "Software"), to deal in the Software without restriction,
+ including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or
+ sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+ provided that the above copyright notice(s) and this permission notice appear in all copies of
+ the Software and that both the above copyright notice(s) and this permission notice appear in
+ supporting documentation.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED
+ IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY
+ DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+ NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ OF THIS SOFTWARE.
+
+ Except as contained in this notice, the name of a copyright holder shall not be used in advertising
+ or otherwise to promote the sale, use or other dealings in this Software without prior written
+ authorization of the copyright holder.
+
+The binary distribution of this product bundles 'Scala Library, Scala Parser, Scala Reflect' which is available
+ under a "3-clause BSD" license. For details see https://github.com/scala/scala/blob/v2.11.7/doc/LICENSE.md)
+
+ This software includes projects with other licenses -- see `doc/LICENSE.md`.
+
+ Copyright (c) 2002-2017 EPFL
+ Copyright (c) 2011-2017 Lightbend, Inc.
+
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the EPFL nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..1ac342e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api-nar/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,612 @@
+nifi-druid-controller-service-api-nar
+Copyright 2015-2017 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+******************
+Apache Software License v2
+******************
+
+The following binary components are provided under the Apache Software License v2
+
+ (ASLv2) Apache HttpComponents Client
+ The following NOTICE information applies:
+ Apache HttpComponents Client
+ Copyright 1999-2016 The Apache Software Foundation
+
+ (ASLv2) Jackson JSON processor
+ The following NOTICE information applies:
+ # Jackson JSON processor
+
+ Jackson is a high-performance, Free/Open Source JSON processing library.
+ It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+ been in development since 2007.
+ It is currently developed by a community of developers, as well as supported
+ commercially by FasterXML.com.
+
+ ## Licensing
+
+ Jackson core and extension components may licensed under different licenses.
+ To find the details that apply to this artifact see the accompanying LICENSE file.
+ For more information, including possible other licensing options, contact
+ FasterXML.com (http://fasterxml.com).
+
+ ## Credits
+
+ A list of contributors may be found from CREDITS file, which is included
+ in some artifacts (usually source distributions); but is always available
+ from the source code management (SCM) system project uses.
+
+ (ASLv2) bytebuffer-collections
+ The following NOTICE information applies:
+ bytebuffer-collections
+ Copyright 2011-2015 Metamarkets Group Inc.
+
+ (ASLv2) Apache Commons IO
+ The following NOTICE information applies:
+ Apache Commons IO
+ Copyright 2002-2016 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Collections
+ The following NOTICE information applies:
+ Apache Commons Collections
+ Copyright 2001-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Lang
+ The following NOTICE information applies:
+ Apache Commons Lang
+ Copyright 2001-2014 The Apache Software Foundation
+
+ This product includes software from the Spring Framework,
+ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+ (ASLv2) Apache Commons Codec
+ The following NOTICE information applies:
+ Apache Commons Codec
+ Copyright 2002-2014 The Apache Software Foundation
+
+ src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+ contains test data from http://aspell.net/test/orig/batch0.tab.
+ Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+ ===============================================================================
+
+ The content of package org.apache.commons.codec.language.bm has been translated
+ from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+ with permission from the original authors.
+ Original source copyright:
+ Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+ (ASLv2) Apache Commons Logging
+ The following NOTICE information applies:
+ Apache Commons Logging
+ Copyright 2003-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons DBCP
+ The following NOTICE information applies:
+ Apache Commons DBCP
+ Copyright 2001-2015 The Apache Software Foundation.
+
+ (ASLv2) Apache Commons Pool
+ The following NOTICE information applies:
+ Apache Commons Pool
+ Copyright 1999-2009 The Apache Software Foundation.
+
+ (ASLv2) Apache Curator
+ The following NOTICE information applies:
+ Curator Framework
+ Copyright 2011-2014 The Apache Software Foundation
+
+ Curator Client
+ Copyright 2011-2014 The Apache Software Foundation
+
+ Curator Recipes
+ Copyright 2011-2014 The Apache Software Foundation
+
+ Curator X-Discovery
+ Copyright 2013-2014 The Apache Software Foundation
+
+ (ASLv2) Druid
+ The following NOTICE information applies:
+ Druid - a distributed column store.
+ Copyright 2012-2016 Metamarkets Group Inc.
+ Copyright 2015-2016 Yahoo! Inc.
+ Copyright 2015-2016 Imply Data, Inc.
+
+ -------------------------------------------------------------------------------
+
+ This product contains a modified version of Andrew Duffy's java-alphanum library
+ * LICENSE:
+ * https://github.com/amjjd/java-alphanum/blob/5c036e2e492cc7f3b7bcdebd46b8f9e2a87927e5/LICENSE.txt (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/amjjd/java-alphanum
+
+ This product contains conjunctive normal form conversion code and a variance aggregator algorithm adapted from Apache Hive
+ * LICENSE:
+ * https://github.com/apache/hive/blob/branch-2.0/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/apache/hive
+
+ This product contains variable length long deserialization code adapted from Apache Lucene
+ * LICENSE:
+ * https://github.com/apache/lucene-solr/blob/master/lucene/LICENSE.txt (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/apache/lucene-solr
+
+ This product contains a modified version of Metamarkets java-util library
+ * LICENSE:
+ * https://github.com/metamx/java-util/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/metamx/java-util
+ * COMMIT TAG:
+ * https://github.com/metamx/java-util/commit/826021f
+
+ This product contains a modified version of TestNG 6.8.7
+ * LICENSE:
+ * http://testng.org/license/ (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * http://testng.org/
+
+ This product contains a modified version of Metamarkets bytebuffer-collections library
+ * LICENSE:
+ * https://github.com/metamx/bytebuffer-collections/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/metamx/bytebuffer-collections
+ * COMMIT TAG:
+ * https://github.com/metamx/bytebuffer-collections/commit/3d1e7c8
+
+ This product contains SQL query planning code adapted from Apache Calcite
+ * LICENSE:
+ * https://github.com/apache/calcite/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://calcite.apache.org/
+
+ This product contains a modified version of Metamarkets extendedset library
+ * LICENSE:
+ * https://github.com/metamx/extendedset/blob/master/LICENSE (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/metamx/extendedset
+ * COMMIT TAG:
+ * https://github.com/metamx/extendedset/commit/c9d647d
+
+ This product contains a modified version of Alessandro Colantonio's CONCISE
+ (COmpressed 'N' Composable Integer SEt) library, extending the functionality of
+ ConciseSet to use IntBuffers.
+ * (c) 2010 Alessandro Colantonio
+ * <ma...@mat.uniroma3.it>
+ * <http://ricerca.mat.uniroma3.it/users/colanton>
+ * LICENSE:
+ * Apache License, Version 2.0
+ * HOMEPAGE:
+ * https://sourceforge.net/projects/concise/
+
+ This product contains a modified version of The Guava Authors's Closer class from Guava library:
+ * LICENSE:
+ * https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c/COPYING (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * https://github.com/google/guava
+ * COMMIT TAG:
+ * https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c
+
+ This product contains code adapted from Apache Hadoop
+ * LICENSE:
+ * https://github.com/apache/hadoop/blob/trunk/LICENSE.txt (Apache License, Version 2.0)
+ * HOMEPAGE:
+ * http://hadoop.apache.org/
+
+ (ASLv2) Emitter
+ The following NOTICE information applies:
+ emitter
+ Copyright 2012-2015 Metamarkets Group Inc.
+
+ (ASLv2) Finagle
+ The following NOTICE information applies:
+ Copyright 2017, Twitter Inc.
+ This software contains portions of code from other open-source projects, including:
+ >>>>>>>>>>>>>>
+ Apache Thrift
+ Copyright 2006-2010 The Apache Software Foundation.
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ (ASLv2) Guava
+ The following NOTICE information applies:
+ Guava
+ Copyright 2015 The Guava Authors
+
+ (ASLv2) Google Guice
+ The following NOTICE information applies:
+ Google Guice - Core Library
+ Copyright 2006-2011 Google, Inc.
+
+ (ASLv2) Jackson JSON processor
+ The following NOTICE information applies:
+ # Jackson JSON processor
+
+ Jackson is a high-performance, Free/Open Source JSON processing library.
+ It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+ been in development since 2007.
+ It is currently developed by a community of developers, as well as supported
+ commercially by FasterXML.com.
+
+ ## Licensing
+
+ Jackson core and extension components may licensed under different licenses.
+ To find the details that apply to this artifact see the accompanying LICENSE file.
+ For more information, including possible other licensing options, contact
+ FasterXML.com (http://fasterxml.com).
+
+ ## Credits
+
+ A list of contributors may be found from CREDITS file, which is included
+ in some artifacts (usually source distributions); but is always available
+ from the source code management (SCM) system project uses.
+
+ (ASLv2) java-util
+ The following NOTICE information applies:
+ java-util
+ Copyright 2011-2017 Metamarkets Group Inc.
+
+ (ASLv2) jdbi
+ The following NOTICE information applies:
+ This product includes software developed by Brian McCallister
+
+ (ASLv2) Joda Time
+ The following NOTICE information applies:
+ This product includes software developed by
+ Joda.org (http://www.joda.org/).
+
+ (ASLv2) Joda Convert
+ The following NOTICE information applies:
+ Joda Convert
+ Copyright 2010-present Stephen Colebourne
+
+ This product includes software developed by
+ Joda.org (http://www.joda.org/).
+
+
+ Joda-Convert includes code from Google Guava, which is licensed as follows:
+
+ Copyright (C) 2011 The Guava Authors
+
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+
+ (ASLv2) Apache log4j
+ The following NOTICE information applies:
+ Apache log4j
+ Copyright 2007 The Apache Software Foundation
+
+ (ASLv2) loglady
+ The following NOTICE information applies:
+ Copyright (c) 2013 Daniel Lundin
+
+ (ASLv2) The Netty Project
+ The following NOTICE information applies:
+
+ The Netty Project
+ =================
+
+ Please visit the Netty web site for more information:
+
+ * http://netty.io/
+
+ Copyright 2014 The Netty Project
+
+ The Netty Project licenses this file to you under the Apache License,
+ version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at:
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ License for the specific language governing permissions and limitations
+ under the License.
+
+ Also, please refer to each LICENSE.<component>.txt file, which is located in
+ the 'license' directory of the distribution file, for the license terms of the
+ components that this product depends on.
+
+ -------------------------------------------------------------------------------
+ This product contains the extensions to Java Collections Framework which has
+ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
+
+ * LICENSE:
+ * license/LICENSE.jsr166y.txt (Public Domain)
+ * HOMEPAGE:
+ * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
+ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+
+ This product contains a modified version of Robert Harder's Public Domain
+ Base64 Encoder and Decoder, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.base64.txt (Public Domain)
+ * HOMEPAGE:
+ * http://iharder.sourceforge.net/current/java/base64/
+
+ This product contains a modified portion of 'Webbit', an event based
+ WebSocket and HTTP server, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.webbit.txt (BSD License)
+ * HOMEPAGE:
+ * https://github.com/joewalnes/webbit
+
+ This product contains a modified portion of 'SLF4J', a simple logging
+ facade for Java, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.slf4j.txt (MIT License)
+ * HOMEPAGE:
+ * http://www.slf4j.org/
+
+ This product contains a modified portion of 'Apache Harmony', an open source
+ Java SE, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.harmony.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * http://archive.apache.org/dist/harmony/
+
+ This product contains a modified portion of 'jbzip2', a Java bzip2 compression
+ and decompression library written by Matthew J. Francis. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.jbzip2.txt (MIT License)
+ * HOMEPAGE:
+ * https://code.google.com/p/jbzip2/
+
+ This product contains a modified portion of 'libdivsufsort', a C API library to construct
+ the suffix array and the Burrows-Wheeler transformed string for any input string of
+ a constant-size alphabet written by Yuta Mori. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.libdivsufsort.txt (MIT License)
+ * HOMEPAGE:
+ * https://github.com/y-256/libdivsufsort
+
+ This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM,
+ which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.jctools.txt (ASL2 License)
+ * HOMEPAGE:
+ * https://github.com/JCTools/JCTools
+
+ This product optionally depends on 'JZlib', a re-implementation of zlib in
+ pure Java, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.jzlib.txt (BSD style License)
+ * HOMEPAGE:
+ * http://www.jcraft.com/jzlib/
+
+ This product optionally depends on 'Compress-LZF', a Java library for encoding and
+ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.compress-lzf.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/ning/compress
+
+ This product optionally depends on 'lz4', a LZ4 Java compression
+ and decompression library written by Adrien Grand. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.lz4.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/jpountz/lz4-java
+
+ This product optionally depends on 'lzma-java', a LZMA Java compression
+ and decompression library, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.lzma-java.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/jponge/lzma-java
+
+ This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression
+ and decompression library written by William Kinney. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.jfastlz.txt (MIT License)
+ * HOMEPAGE:
+ * https://code.google.com/p/jfastlz/
+
+ This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data
+ interchange format, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.protobuf.txt (New BSD License)
+ * HOMEPAGE:
+ * https://github.com/google/protobuf
+
+ This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
+ a temporary self-signed X.509 certificate when the JVM does not provide the
+ equivalent functionality. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.bouncycastle.txt (MIT License)
+ * HOMEPAGE:
+ * http://www.bouncycastle.org/
+
+ This product optionally depends on 'Snappy', a compression library produced
+ by Google Inc, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.snappy.txt (New BSD License)
+ * HOMEPAGE:
+ * https://github.com/google/snappy
+
+ This product optionally depends on 'JBoss Marshalling', an alternative Java
+ serialization API, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1)
+ * HOMEPAGE:
+ * http://www.jboss.org/jbossmarshalling
+
+ This product optionally depends on 'Caliper', Google's micro-
+ benchmarking framework, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.caliper.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/google/caliper
+
+ This product optionally depends on 'Apache Commons Logging', a logging
+ framework, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.commons-logging.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * http://commons.apache.org/logging/
+
+ This product optionally depends on 'Apache Log4J', a logging framework, which
+ can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.log4j.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * http://logging.apache.org/log4j/
+
+ This product optionally depends on 'Aalto XML', an ultra-high performance
+ non-blocking XML processor, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.aalto-xml.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * http://wiki.fasterxml.com/AaltoHome
+
+ This product contains a modified version of 'HPACK', a Java implementation of
+ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.hpack.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/twitter/hpack
+
+ This product contains a modified portion of 'Apache Commons Lang', a Java library
+ provides utilities for the java.lang API, which can be obtained at:
+
+ * LICENSE:
+ * license/LICENSE.commons-lang.txt (Apache License 2.0)
+ * HOMEPAGE:
+ * https://commons.apache.org/proper/commons-lang/
+
+ This product contains a forked and modified version of Tomcat Native
+
+ * LICENSE:
+ * ASL2
+ * HOMEPAGE:
+ * http://tomcat.apache.org/native-doc/
+ * https://svn.apache.org/repos/asf/tomcat/native/
+
+ (ASLv2) opencsv (net.sf.opencsv:opencsv:2.3)
+
+ (ASLv2) scala_util
+ The following NOTICE information applies:
+ Copyright 2012 Metamarkets Group Inc.
+
+ (ASLv2) server-metrics
+ The following NOTICE information applies:
+ server-metrics
+ Copyright 2011-2015 Metamarkets Group Inc.
+
+ (ASLv2) Apache ZooKeeper
+ The following NOTICE information applies:
+ Apache ZooKeeper
+ Copyright 2009-2012 The Apache Software Foundation
+
+************************
+Eclipse Public License 1.0
+************************
+
+The following binary components are provided under the Eclipse Public License 1.0. See project link for details.
+
+ (EPL 1.0)(MPL 2.0) H2 Database (com.h2database:h2:jar:1.3.176 - http://www.h2database.com/html/license.html)
+
+
+************************
+Mozilla Public License 2.0
+************************
+
+The following binary components are provided under the Mozilla Public License 2.0. See project link for details.
+
+ (MPL 2.0) Rhino (org.mozilla:rhino:jar:1.7R5 - https://developer.mozilla.org/en-US/docs/Mozilla/Projects/Rhino/Download_Rhino)
+
+
+************************
+Common Development and Distribution License 1.1
+************************
+
+The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details.
+
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-entity-filtering (org.glassfish.jersey.ext:jersey-entity-filtering:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-hk2 (org.glassfish.jersey.inject:jersey-hk2:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-media-jaxb (org.glassfish.jersey.media:jersey-media-jaxb:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-media-json-jackson (org.glassfish.jersey.media:jersey-media-json-jackson:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-media-multipart (org.glassfish.jersey.media:jersey-media-multipart:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-server (org.glassfish.jersey.core:jersey-server:jar:2.26 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2 (org.glassfish.hk2:hk2:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2-api (org.glassfish.hk2:hk2-api:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2-utils (org.glassfish.hk2:hk2-utils:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2-locator (org.glassfish.hk2:hk2-locator:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2-config (org.glassfish.hk2:hk2-config:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2-core (org.glassfish.hk2:hk2-core:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) hk2-runlevel (org.glassfish.hk2:hk2-runlevel:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) config-types (org.glassfish.hk2:config-types:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) class-model (org.glassfish.hk2:class-model:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) asm-all-repackaged (org.glassfish.hk2.external:asm-all-repackaged:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) aopalliance-repackaged (org.glassfish.hk2.external:aopalliance-repackaged:jar:2.5.0-b42 - https://javaee.github.io/glassfish/)
+ (CDDL 1.1) (GPL2 w/ CPE) javax.inject:1 as OSGi bundle (org.glassfish.hk2.external:javax.inject:jar:2.4.0-b25 - https://hk2.java.net/external/javax.inject)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.19 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.19 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.19 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.22.1 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.22.1 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-media-multipart (org.glassfish.jersey.media:jersey-media-multipart:jar:2.22.1 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.22.1 - https://jersey.github.io/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-client (com.sun.jersey:jersey-client:jar:1.9 - https://jersey.java.net/jersey-client/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-core (com.sun.jersey:jersey-core:jar:1.9 - https://jersey.java.net/jersey-core/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-json (com.sun.jersey:jersey-json:jar:1.9 - https://jersey.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.9 - https://jersey.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-guice (com.sun.jersey.contribs:jersey-guice:jar:1.9 - https://jersey.java.net/)
+
+
+ (CDDL 1.1) (GPL2 w/ CPE) Old JAXB Runtime (com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 - http://jaxb.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) Java Architecture For XML Binding (javax.xml.bind:jaxb-api:jar:2.2.2 - https://jaxb.dev.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) MIME Streaming Extension (org.jvnet.mimepull:mimepull:jar:1.9.3 - http://mimepull.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail)
+ (CDDL 1.1) (GPL2 w/ CPE) JSP Implementation (org.glassfish.web:javax.servlet.jsp:jar:2.3.2 - http://jsp.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages (TM) TagLib Implementation (org.glassfish.web:javax.servlet.jsp.jstl:jar:1.2.2 - http://jstl.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 (org.glassfish:javax.el:jar:3.0.0 - http://el-spec.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages(TM) API (javax.servlet.jsp:javax.servlet.jsp-api:jar:2.3.1 - http://jsp.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 API (javax.el:javax.el-api:jar:3.0.0 - http://uel-spec.java.net)
+
+*****************
+Public Domain
+*****************
+
+The following binary components are provided to the 'Public Domain'. See project link for details.
+
+ (Public Domain) AOP Alliance 1.0 (http://aopalliance.sourceforge.net/)
+
+The following binary components are provided under the Creative Commons Zero license version 1.0. See project link for details.
+
+ (CC0v1.0) JSR166e for Twitter (com.twitter:jsr166e:jar:1.1.0 - https://github.com/twitter/jsr166e)
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
new file mode 100644
index 0000000..7883e3a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
@@ -0,0 +1,137 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-bundle</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-druid-controller-service-api</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>tranquility-core_2.11</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>io.druid</groupId>
+ <artifactId>druid-console</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.druid</groupId>
+ <artifactId>druid-aws-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>c3p0</groupId>
+ <artifactId>c3p0</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.tesla.aether</groupId>
+ <artifactId>tesla-aether</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.aether</groupId>
+ <artifactId>aether-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.maxmind.geoip2</groupId>
+ <artifactId>geoip2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.druid</groupId>
+ <artifactId>druid-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>druid-server</artifactId>
+ <version>${druid.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>druid-processing</artifactId>
+ <version>${druid.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.airlift</groupId>
+ <artifactId>airline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ <version>1.3.9-1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/src/main/java/org/apache/nifi/controller/api/druid/DruidTranquilityService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/src/main/java/org/apache/nifi/controller/api/druid/DruidTranquilityService.java b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/src/main/java/org/apache/nifi/controller/api/druid/DruidTranquilityService.java
new file mode 100644
index 0000000..ca83207
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/src/main/java/org/apache/nifi/controller/api/druid/DruidTranquilityService.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.api.druid;
+
+import java.util.Map;
+
+import org.apache.nifi.controller.ControllerService;
+
+import com.metamx.tranquility.tranquilizer.Tranquilizer;
+
+public interface DruidTranquilityService extends ControllerService {
+ Tranquilizer<Map<String, Object>> getTranquilizer();
+
+ String getTransitUri();
+}
\ No newline at end of file
[45/50] nifi git commit: NIFI-5043: TailFile in Multifile mode should
not open new readers in onTrigger
Posted by al...@apache.org.
NIFI-5043: TailFile in Multifile mode should not open new readers in onTrigger
This closes #2606.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/1bed849d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/1bed849d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/1bed849d
Branch: refs/heads/HDF-3.1-maint
Commit: 1bed849dcb83ac24de0e0629ef4e3db2fe29d2c4
Parents: 8f988c3
Author: Marco Gaido <ma...@gmail.com>
Authored: Thu Apr 5 13:44:55 2018 +0200
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Apr 13 09:34:24 2018 -0400
----------------------------------------------------------------------
.../nifi/processors/standard/TailFile.java | 25 +++++++++++++-------
1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/1bed849d/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
index 2234265..01893e9 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
@@ -282,11 +282,7 @@ public class TailFile extends AbstractProcessor {
this.requireStateLookup = true;
}
- @OnScheduled
- public void recoverState(final ProcessContext context) throws IOException {
- // set isMultiChanging
- isMultiChanging.set(context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue()));
-
+ private List<String> lookup(final ProcessContext context) {
// set last lookup to now
lastLookup.set(new Date().getTime());
@@ -298,13 +294,21 @@ public class TailFile extends AbstractProcessor {
if(context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue())) {
filesToTail.addAll(getFilesToTail(context.getProperty(BASE_DIRECTORY).evaluateAttributeExpressions().getValue(),
- context.getProperty(FILENAME).evaluateAttributeExpressions().getValue(),
- context.getProperty(RECURSIVE).asBoolean(),
- maxAge));
+ context.getProperty(FILENAME).evaluateAttributeExpressions().getValue(),
+ context.getProperty(RECURSIVE).asBoolean(),
+ maxAge));
} else {
filesToTail.add(context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
}
+ return filesToTail;
+ }
+
+ @OnScheduled
+ public void recoverState(final ProcessContext context) throws IOException {
+ // set isMultiChanging
+ isMultiChanging.set(context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue()));
+ List<String> filesToTail = lookup(context);
final Scope scope = getStateScope(context);
final StateMap stateMap = context.getStateManager().getState(scope);
@@ -569,7 +573,10 @@ public class TailFile extends AbstractProcessor {
long timeSinceLastLookup = new Date().getTime() - lastLookup.get();
if(timeSinceLastLookup > context.getProperty(LOOKUP_FREQUENCY).asTimePeriod(TimeUnit.MILLISECONDS)) {
try {
- recoverState(context);
+ final List<String> filesToTail = lookup(context);
+ final Scope scope = getStateScope(context);
+ final StateMap stateMap = context.getStateManager().getState(scope);
+ initStates(filesToTail, stateMap.toMap(), false);
} catch (IOException e) {
getLogger().error("Exception raised while attempting to recover state about where the tailing last left off", e);
context.yield();
[02/50] nifi git commit: NIFI-4428: Replaced JSON input with
RecordReader using per-record flowfiles NIFI-4428: Added initial L&N,
fixed dependency hierarchy
Posted by al...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
new file mode 100644
index 0000000..d852fe1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/pom.xml
@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-bundle</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-druid-controller-service</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service-api</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>tranquility-core_2.11</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/java/org/apache/nifi/controller/druid/DruidTranquilityController.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/java/org/apache/nifi/controller/druid/DruidTranquilityController.java b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/java/org/apache/nifi/controller/druid/DruidTranquilityController.java
new file mode 100644
index 0000000..e5af8ea
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/java/org/apache/nifi/controller/druid/DruidTranquilityController.java
@@ -0,0 +1,614 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.druid;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnDisabled;
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.controller.api.druid.DruidTranquilityService;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import com.metamx.common.Granularity;
+import com.metamx.tranquility.beam.Beam;
+import com.metamx.tranquility.beam.ClusteredBeamTuning;
+import com.metamx.tranquility.druid.DruidBeamConfig;
+import com.metamx.tranquility.druid.DruidBeams;
+import com.metamx.tranquility.druid.DruidDimensions;
+import com.metamx.tranquility.druid.DruidEnvironment;
+import com.metamx.tranquility.druid.DruidLocation;
+import com.metamx.tranquility.druid.DruidRollup;
+import com.metamx.tranquility.tranquilizer.Tranquilizer;
+import com.metamx.tranquility.typeclass.Timestamper;
+
+import io.druid.data.input.impl.TimestampSpec;
+import io.druid.granularity.QueryGranularity;
+import io.druid.query.aggregation.AggregatorFactory;
+import io.druid.query.aggregation.CountAggregatorFactory;
+import io.druid.query.aggregation.DoubleMaxAggregatorFactory;
+import io.druid.query.aggregation.DoubleMinAggregatorFactory;
+import io.druid.query.aggregation.DoubleSumAggregatorFactory;
+import io.druid.query.aggregation.LongMaxAggregatorFactory;
+import io.druid.query.aggregation.LongMinAggregatorFactory;
+import io.druid.query.aggregation.LongSumAggregatorFactory;
+import org.joda.time.DateTime;
+import org.joda.time.Period;
+
+
+@Tags({"Druid", "Timeseries", "OLAP", "ingest"})
+@CapabilityDescription("Asynchronously sends flowfiles to Druid Indexing Task using Tranquility API. "
+ + "If aggregation and roll-up of data is required, an Aggregator JSON descriptor needs to be provided."
+ + "Details on how describe aggregation using JSON can be found at: http://druid.io/docs/latest/querying/aggregations.html")
+public class DruidTranquilityController extends AbstractControllerService implements DruidTranquilityService {
+ private final static String FIREHOSE_PATTERN = "druid:firehose:%s";
+
+ private final static AllowableValue PT1M = new AllowableValue("PT1M", "1 minute", "1 minute");
+ private final static AllowableValue PT10M = new AllowableValue("PT10M", "10 minutes", "10 minutes");
+ private final static AllowableValue PT60M = new AllowableValue("PT60M", "60 minutes", "1 hour");
+
+ private final static List<String> TIME_ORDINALS = Arrays.asList("SECOND", "MINUTE", "FIVE_MINUTE", "TEN_MINUTE", "FIFTEEN_MINUTE", "HOUR", "SIX_HOUR", "DAY", "WEEK", "MONTH", "YEAR");
+
+ private Tranquilizer tranquilizer = null;
+ private String transitUri = "";
+
+ public static final PropertyDescriptor DATASOURCE = new PropertyDescriptor.Builder()
+ .name("druid-cs-data-source")
+ .displayName("Druid Data Source")
+ .description("A data source is the Druid equivalent of a database table.")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .required(true)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor ZOOKEEPER_CONNECTION_STRING = new PropertyDescriptor.Builder()
+ .name("druid-cs-zk-connect-string")
+ .displayName("Zookeeper Connection String")
+ .description("A comma-separated list of host:port pairs, each corresponding to a ZooKeeper server. Ex: localhost:2181")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor ZOOKEEPER_RETRY_BASE_SLEEP_TIME = new PropertyDescriptor.Builder()
+ .name("druid-cs-zk-retry-base-sleep")
+ .displayName("Zookeeper Retry Base Sleep Time")
+ .description("When a connection to Zookeeper needs to be retried, this property specifies the amount of time (in milliseconds) to wait at first before retrying.")
+ .required(true)
+ .defaultValue("1000")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor ZOOKEEPER_RETRY_MAX_RETRIES = new PropertyDescriptor.Builder()
+ .name("druid-cs-zk-retry-max-retries")
+ .displayName("Zookeeper Retry Max Retries")
+ .description("When a connection to Zookeeper needs to be retried, this property specifies how many times to attempt reconnection.")
+ .required(true)
+ .defaultValue("20")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor ZOOKEEPER_RETRY_SLEEP_TIME = new PropertyDescriptor.Builder()
+ .name("druid-cs-zk-retry-sleep")
+ .displayName("Zookeeper Retry Sleep Time")
+ .description("When a connection to Zookeeper needs to be retried, this property specifies the amount of time to sleep (in milliseconds) between retries.")
+ .required(true)
+ .defaultValue("30000")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor DRUID_INDEX_SERVICE_PATH = new PropertyDescriptor.Builder()
+ .name("druid-cs-index-service-path")
+ .displayName("Index Service Path")
+ .description("Druid Index Service path as defined via the Druid Overlord druid.service property.")
+ .required(true)
+ .defaultValue("druid/overlord")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor DRUID_DISCOVERY_PATH = new PropertyDescriptor.Builder()
+ .name("druid-cs-discovery-path")
+ .displayName("Discovery Path")
+ .description("Druid Discovery Path as configured in Druid Common druid.discovery.curator.path property")
+ .required(true)
+ .defaultValue("/druid/discovery")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor CLUSTER_PARTITIONS = new PropertyDescriptor.Builder()
+ .name("druid-cs-cluster-partitions")
+ .displayName("Cluster Partitions")
+ .description("The number of partitions in the Druid cluster.")
+ .required(true)
+ .defaultValue("1")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CLUSTER_REPLICATION = new PropertyDescriptor.Builder()
+ .name("druid-cs-cluster-replication")
+ .displayName("Cluster Replication")
+ .description("The replication factor for the Druid cluster.")
+ .required(true)
+ .defaultValue("1")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor TIMESTAMP_FIELD = new PropertyDescriptor.Builder()
+ .name("druid-cs-timestamp-field")
+ .displayName("Timestamp field")
+ .description("The name of the field that will be used as the timestamp. Should be in ISO8601 format.")
+ .required(true)
+ .defaultValue("timestamp")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor AGGREGATOR_JSON = new PropertyDescriptor.Builder()
+ .name("druid-cs-aggregators-descriptor")
+ .displayName("Aggregator JSON")
+ .description("Tranquility-compliant JSON string that defines what aggregators to apply on ingest."
+ + "Example: "
+ + "["
+ + "{"
+ + "\t\"type\" : \"count\","
+ + "\t\"name\" : \"count\","
+ + "},"
+ + "{"
+ + "\t\"name\" : \"value_sum\","
+ + "\t\"type\" : \"doubleSum\","
+ + "\t\"fieldName\" : \"value\""
+ + "},"
+ + "{"
+ + "\t\"fieldName\" : \"value\","
+ + "\t\"name\" : \"value_min\","
+ + "\t\"type\" : \"doubleMin\""
+ + "},"
+ + "{"
+ + "\t\"type\" : \"doubleMax\","
+ + "\t\"name\" : \"value_max\","
+ + "\t\"fieldName\" : \"value\""
+ + "}"
+ + "]")
+ .required(true)
+ .addValidator((subject, value, context) -> { // Non-empty and valid JSON validator
+ if (value == null || value.isEmpty()) {
+ return new ValidationResult.Builder().subject(subject).input(value).valid(false).explanation(subject + " cannot be empty").build();
+ }
+ try {
+ DruidTranquilityController.parseJsonString(value);
+ return new ValidationResult.Builder().subject(subject).input(value).valid(true).build();
+ } catch (IllegalArgumentException iae) {
+ return new ValidationResult.Builder().subject(subject).input(value).valid(false).explanation(subject + " is not valid Aggregator JSON").build();
+ }
+ })
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor DIMENSIONS_LIST = new PropertyDescriptor.Builder()
+ .name("druid-cs-dimensions-list")
+ .displayName("Dimension Fields")
+ .description("A comma separated list of field names that will be stored as dimensions on ingest.")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor SEGMENT_GRANULARITY = new PropertyDescriptor.Builder()
+ .name("druid-cs-segment-granularity")
+ .displayName("Segment Granularity")
+ .description("Time unit by which to group and aggregate/rollup events. The value must be at least as large as the value of Query Granularity.")
+ .required(true)
+ .allowableValues("NONE", "SECOND", "MINUTE", "TEN_MINUTE", "HOUR", "DAY", "MONTH", "YEAR")
+ .defaultValue("TEN_MINUTE")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor QUERY_GRANULARITY = new PropertyDescriptor.Builder()
+ .name("druid-cs-query-granularity")
+ .displayName("Query Granularity")
+ .description("Time unit by which to group and aggregate/rollup events. The value must be less than or equal to the value of Segment Granularity.")
+ .required(true)
+ .allowableValues("NONE", "SECOND", "MINUTE", "FIFTEEN_MINUTE", "THIRTY_MINUTE", "HOUR", "DAY", "MONTH", "YEAR")
+ .defaultValue("MINUTE")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor INDEX_RETRY_PERIOD = new PropertyDescriptor.Builder()
+ .name("druid-cs-index-retry-period")
+ .displayName("Index Retry Period")
+ .description("Grace period to allow late arriving events for real time ingest.")
+ .required(true)
+ .allowableValues(PT1M, PT10M, PT60M)
+ .defaultValue(PT10M.getValue())
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor WINDOW_PERIOD = new PropertyDescriptor.Builder()
+ .name("druid-cs-window-period")
+ .displayName("Late Event Grace Period")
+ .description("Grace period to allow late arriving events for real time ingest.")
+ .required(true)
+ .allowableValues(PT1M, PT10M, PT60M)
+ .defaultValue(PT10M.getValue())
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor MAX_BATCH_SIZE = new PropertyDescriptor.Builder()
+ .name("druid-cs-batch-size")
+ .displayName("Batch Size")
+ .description("Maximum number of messages to send at once.")
+ .required(true)
+ .defaultValue("2000")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor MAX_PENDING_BATCHES = new PropertyDescriptor.Builder()
+ .name("druid-cs-max-pending-batches")
+ .displayName("Max Pending Batches")
+ .description("Maximum number of batches that may be in flight before service blocks and waits for one to finish.")
+ .required(true)
+ .defaultValue("5")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor LINGER_MILLIS = new PropertyDescriptor.Builder()
+ .name("druid-cs-linger-millis")
+ .displayName("Linger (milliseconds)")
+ .description("Wait this long for batches to collect more messages (up to Batch Size) before sending them. "
+ + "Set to zero to disable waiting. "
+ + "Set to -1 to always wait for complete batches before sending. ")
+ .required(true)
+ .defaultValue("1000")
+ .addValidator(StandardValidators.INTEGER_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+
+ private static final List<PropertyDescriptor> properties;
+
+ private volatile CuratorFramework curator;
+ private volatile int zkBaseSleepMillis;
+ private volatile int zkMaxRetries;
+ private volatile int zkSleepMillis;
+
+ static {
+ final List<PropertyDescriptor> props = new ArrayList<>();
+ props.add(DATASOURCE);
+ props.add(ZOOKEEPER_CONNECTION_STRING);
+ props.add(ZOOKEEPER_RETRY_BASE_SLEEP_TIME);
+ props.add(ZOOKEEPER_RETRY_MAX_RETRIES);
+ props.add(ZOOKEEPER_RETRY_SLEEP_TIME);
+ props.add(DRUID_INDEX_SERVICE_PATH);
+ props.add(DRUID_DISCOVERY_PATH);
+ props.add(CLUSTER_PARTITIONS);
+ props.add(CLUSTER_REPLICATION);
+ props.add(DIMENSIONS_LIST);
+ props.add(AGGREGATOR_JSON);
+ props.add(SEGMENT_GRANULARITY);
+ props.add(QUERY_GRANULARITY);
+ props.add(WINDOW_PERIOD);
+ props.add(TIMESTAMP_FIELD);
+ props.add(MAX_BATCH_SIZE);
+ props.add(MAX_PENDING_BATCHES);
+ props.add(LINGER_MILLIS);
+
+ properties = Collections.unmodifiableList(props);
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ Set<ValidationResult> results = new HashSet<>();
+ final String segmentGranularity = validationContext.getProperty(SEGMENT_GRANULARITY).getValue();
+ final String queryGranularity = validationContext.getProperty(QUERY_GRANULARITY).getValue();
+
+ // Verify that segment granularity is as least as large as query granularity
+ if (TIME_ORDINALS.indexOf(segmentGranularity) < TIME_ORDINALS.indexOf(queryGranularity)) {
+ results.add(new ValidationResult.Builder().valid(false).explanation(
+ "Segment Granularity must be at least as large as Query Granularity").build());
+ }
+
+ return results;
+ }
+
+ @OnEnabled
+ public void onConfigured(final ConfigurationContext context) {
+ ComponentLog log = getLogger();
+ log.info("Starting Druid Tranquility Controller Service...");
+
+ final String dataSource = context.getProperty(DATASOURCE).evaluateAttributeExpressions().getValue();
+ final String zkConnectString = context.getProperty(ZOOKEEPER_CONNECTION_STRING).evaluateAttributeExpressions().getValue();
+ zkBaseSleepMillis = context.getProperty(ZOOKEEPER_RETRY_BASE_SLEEP_TIME).evaluateAttributeExpressions().asInteger();
+ zkMaxRetries = context.getProperty(ZOOKEEPER_RETRY_BASE_SLEEP_TIME).evaluateAttributeExpressions().asInteger();
+ zkSleepMillis = context.getProperty(ZOOKEEPER_RETRY_SLEEP_TIME).evaluateAttributeExpressions().asInteger();
+ final String indexService = context.getProperty(DRUID_INDEX_SERVICE_PATH).evaluateAttributeExpressions().getValue();
+ final String discoveryPath = context.getProperty(DRUID_DISCOVERY_PATH).evaluateAttributeExpressions().getValue();
+ final int clusterPartitions = context.getProperty(CLUSTER_PARTITIONS).evaluateAttributeExpressions().asInteger();
+ final int clusterReplication = context.getProperty(CLUSTER_REPLICATION).evaluateAttributeExpressions().asInteger();
+ final String timestampField = context.getProperty(TIMESTAMP_FIELD).evaluateAttributeExpressions().getValue();
+ final String segmentGranularity = context.getProperty(SEGMENT_GRANULARITY).getValue();
+ final String queryGranularity = context.getProperty(QUERY_GRANULARITY).getValue();
+ final String windowPeriod = context.getProperty(WINDOW_PERIOD).getValue();
+ final String indexRetryPeriod = context.getProperty(INDEX_RETRY_PERIOD).getValue();
+ final String aggregatorJSON = context.getProperty(AGGREGATOR_JSON).evaluateAttributeExpressions().getValue();
+ final String dimensionsStringList = context.getProperty(DIMENSIONS_LIST).getValue();
+ final int maxBatchSize = context.getProperty(MAX_BATCH_SIZE).evaluateAttributeExpressions().asInteger();
+ final int maxPendingBatches = context.getProperty(MAX_PENDING_BATCHES).evaluateAttributeExpressions().asInteger();
+ final int lingerMillis = context.getProperty(LINGER_MILLIS).evaluateAttributeExpressions().asInteger();
+
+ transitUri = String.format(FIREHOSE_PATTERN, dataSource) + ";indexServicePath=" + indexService;
+
+ final List<String> dimensions = getDimensions(dimensionsStringList);
+ final List<AggregatorFactory> aggregator = getAggregatorList(aggregatorJSON);
+
+ final Timestamper<Map<String, Object>> timestamper = new Timestamper<Map<String, Object>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public DateTime timestamp(Map<String, Object> theMap) {
+ return new DateTime(theMap.get(timestampField));
+ }
+ };
+
+ Iterator<AggregatorFactory> aggIterator = aggregator.iterator();
+ AggregatorFactory currFactory;
+ log.debug("Number of Aggregations Defined: {}", new Object[]{aggregator.size()});
+ while (aggIterator.hasNext()) {
+ currFactory = aggIterator.next();
+ log.debug("Verifying Aggregator Definition\n\tAggregator Name: {}\n\tAggregator Type: {}\n\tAggregator Req Fields: {}",
+ new Object[]{currFactory.getName(), currFactory.getTypeName(), currFactory.requiredFields()});
+ }
+ // Tranquility uses ZooKeeper (through Curator) for coordination.
+ curator = getCurator(zkConnectString);
+ curator.start();
+
+ // The JSON serialization of your object must have a timestamp field in a format that Druid understands. By default,
+ // Druid expects the field to be called "timestamp" and to be an ISO8601 timestamp.
+ final TimestampSpec timestampSpec = new TimestampSpec(timestampField, "auto", null);
+
+ final Beam<Map<String, Object>> beam = buildBeam(dataSource, indexService, discoveryPath, clusterPartitions, clusterReplication,
+ segmentGranularity, queryGranularity, windowPeriod, indexRetryPeriod, dimensions, aggregator, timestamper, timestampSpec);
+
+ tranquilizer = buildTranquilizer(maxBatchSize, maxPendingBatches, lingerMillis, beam);
+
+ tranquilizer.start();
+ }
+
+ Tranquilizer<Map<String, Object>> buildTranquilizer(int maxBatchSize, int maxPendingBatches, int lingerMillis, Beam<Map<String, Object>> beam) {
+ return Tranquilizer.builder()
+ .maxBatchSize(maxBatchSize)
+ .maxPendingBatches(maxPendingBatches)
+ .lingerMillis(lingerMillis)
+ .blockOnFull(true)
+ .build(beam);
+ }
+
+ Beam<Map<String, Object>> buildBeam(String dataSource, String indexService, String discoveryPath, int clusterPartitions, int clusterReplication,
+ String segmentGranularity, String queryGranularity, String windowPeriod, String indexRetryPeriod, List<String> dimensions,
+ List<AggregatorFactory> aggregator, Timestamper<Map<String, Object>> timestamper, TimestampSpec timestampSpec) {
+ return DruidBeams.builder(timestamper)
+ .curator(curator)
+ .discoveryPath(discoveryPath)
+ .location(DruidLocation.create(DruidEnvironment.create(indexService, FIREHOSE_PATTERN), dataSource))
+ .timestampSpec(timestampSpec)
+ .rollup(DruidRollup.create(DruidDimensions.specific(dimensions), aggregator, QueryGranularity.fromString(queryGranularity)))
+ .tuning(
+ ClusteredBeamTuning
+ .builder()
+ .segmentGranularity(getGranularity(segmentGranularity))
+ .windowPeriod(new Period(windowPeriod))
+ .partitions(clusterPartitions)
+ .replicants(clusterReplication)
+ .build()
+ )
+ .druidBeamConfig(
+ DruidBeamConfig
+ .builder()
+ .indexRetryPeriod(new Period(indexRetryPeriod))
+ .build())
+ .buildBeam();
+ }
+
+ @OnDisabled
+ public void onDisabled() {
+ if (tranquilizer != null) {
+ tranquilizer.flush();
+ tranquilizer.stop();
+ tranquilizer = null;
+ }
+
+ if (curator != null) {
+ curator.close();
+ curator = null;
+ }
+ }
+
+ public Tranquilizer getTranquilizer() {
+ return tranquilizer;
+ }
+
+ CuratorFramework getCurator(String zkConnectString) {
+ return CuratorFrameworkFactory
+ .builder()
+ .connectString(zkConnectString)
+ .retryPolicy(new ExponentialBackoffRetry(zkBaseSleepMillis, zkMaxRetries, zkSleepMillis))
+ .build();
+ }
+
+ @Override
+ public String getTransitUri() {
+ return transitUri;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static List<Map<String, String>> parseJsonString(String aggregatorJson) {
+ if (aggregatorJson == null) {
+ return Collections.EMPTY_LIST;
+ }
+ ObjectMapper mapper = new ObjectMapper();
+ final List<Map<String, String>> aggSpecList;
+ try {
+ aggSpecList = mapper.readValue(aggregatorJson, List.class);
+ return aggSpecList;
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Exception while parsing the aggregrator JSON");
+ }
+ }
+
+ private List<String> getDimensions(String dimensionStringList) {
+ List<String> dimensionList = new ArrayList<>();
+ if (dimensionStringList != null) {
+ Arrays.stream(dimensionStringList.split(","))
+ .filter(StringUtils::isNotBlank)
+ .map(String::trim)
+ .forEach(dimensionList::add);
+ }
+ return dimensionList;
+ }
+
+ private List<AggregatorFactory> getAggregatorList(String aggregatorJSON) {
+ List<AggregatorFactory> aggregatorList = new LinkedList<>();
+ List<Map<String, String>> aggregatorInfo = parseJsonString(aggregatorJSON);
+ for (Map<String, String> aggregator : aggregatorInfo) {
+
+ if (aggregator.get("type").equalsIgnoreCase("count")) {
+ aggregatorList.add(getCountAggregator(aggregator));
+ } else if (aggregator.get("type").equalsIgnoreCase("doublesum")) {
+ aggregatorList.add(getDoubleSumAggregator(aggregator));
+ } else if (aggregator.get("type").equalsIgnoreCase("doublemax")) {
+ aggregatorList.add(getDoubleMaxAggregator(aggregator));
+ } else if (aggregator.get("type").equalsIgnoreCase("doublemin")) {
+ aggregatorList.add(getDoubleMinAggregator(aggregator));
+ } else if (aggregator.get("type").equalsIgnoreCase("longsum")) {
+ aggregatorList.add(getLongSumAggregator(aggregator));
+ } else if (aggregator.get("type").equalsIgnoreCase("longmax")) {
+ aggregatorList.add(getLongMaxAggregator(aggregator));
+ } else if (aggregator.get("type").equalsIgnoreCase("longmin")) {
+ aggregatorList.add(getLongMinAggregator(aggregator));
+ }
+ }
+
+ return aggregatorList;
+ }
+
+ private AggregatorFactory getLongMinAggregator(Map<String, String> map) {
+ return new LongMinAggregatorFactory(map.get("name"), map.get("fieldName"));
+ }
+
+ private AggregatorFactory getLongMaxAggregator(Map<String, String> map) {
+ return new LongMaxAggregatorFactory(map.get("name"), map.get("fieldName"));
+ }
+
+ private AggregatorFactory getLongSumAggregator(Map<String, String> map) {
+ return new LongSumAggregatorFactory(map.get("name"), map.get("fieldName"));
+ }
+
+ private AggregatorFactory getDoubleMinAggregator(Map<String, String> map) {
+ return new DoubleMinAggregatorFactory(map.get("name"), map.get("fieldName"));
+ }
+
+ private AggregatorFactory getDoubleMaxAggregator(Map<String, String> map) {
+ return new DoubleMaxAggregatorFactory(map.get("name"), map.get("fieldName"));
+ }
+
+ private AggregatorFactory getDoubleSumAggregator(Map<String, String> map) {
+ return new DoubleSumAggregatorFactory(map.get("name"), map.get("fieldName"));
+ }
+
+ private AggregatorFactory getCountAggregator(Map<String, String> map) {
+ return new CountAggregatorFactory(map.get("name"));
+ }
+
+ private Granularity getGranularity(String granularityString) {
+ Granularity granularity = Granularity.HOUR;
+
+ switch (granularityString) {
+ case "SECOND":
+ granularity = Granularity.SECOND;
+ break;
+ case "MINUTE":
+ granularity = Granularity.MINUTE;
+ break;
+ case "FIVE_MINUTE":
+ granularity = Granularity.FIVE_MINUTE;
+ break;
+ case "TEN_MINUTE":
+ granularity = Granularity.TEN_MINUTE;
+ break;
+ case "FIFTEEN_MINUTE":
+ granularity = Granularity.FIFTEEN_MINUTE;
+ break;
+ case "HOUR":
+ granularity = Granularity.HOUR;
+ break;
+ case "SIX_HOUR":
+ granularity = Granularity.SIX_HOUR;
+ break;
+ case "DAY":
+ granularity = Granularity.DAY;
+ break;
+ case "WEEK":
+ granularity = Granularity.WEEK;
+ break;
+ case "MONTH":
+ granularity = Granularity.MONTH;
+ break;
+ case "YEAR":
+ granularity = Granularity.YEAR;
+ break;
+ default:
+ break;
+ }
+ return granularity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
new file mode 100644
index 0000000..201af17
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.nifi.controller.druid.DruidTranquilityController
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/test/java/org/apache/nifi/controller/druid/DruidTranquilityControllerTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/test/java/org/apache/nifi/controller/druid/DruidTranquilityControllerTest.java b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/test/java/org/apache/nifi/controller/druid/DruidTranquilityControllerTest.java
new file mode 100644
index 0000000..56c2616
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service/src/test/java/org/apache/nifi/controller/druid/DruidTranquilityControllerTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.druid;
+
+
+import com.metamx.tranquility.tranquilizer.Tranquilizer;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.api.druid.DruidTranquilityService;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+
+public class DruidTranquilityControllerTest {
+
+ private TestRunner runner;
+ private MockDruidTranquilityController service;
+
+ @Before
+ public void setup() throws Exception {
+ runner = TestRunners.newTestRunner(TestControllerServiceProcessor.class);
+ service = new MockDruidTranquilityController();
+ runner.addControllerService("Client Service", service);
+ }
+
+ @Test
+ public void testValid() {
+ runner.assertNotValid(service);
+ runner.setProperty(service, DruidTranquilityController.DATASOURCE, "test");
+ runner.assertNotValid(service);
+ runner.setProperty(service, DruidTranquilityController.ZOOKEEPER_CONNECTION_STRING, "localhost:2181");
+ runner.assertNotValid(service);
+ runner.setProperty(service, DruidTranquilityController.AGGREGATOR_JSON, "[{\"type\": \"count\", \"name\": \"count\"}]");
+ runner.assertNotValid(service);
+ runner.setProperty(service, DruidTranquilityController.DIMENSIONS_LIST, "dim1,dim2");
+ runner.assertValid(service);
+ }
+
+ public static class MockDruidTranquilityController extends DruidTranquilityController {
+
+ Tranquilizer t = mock(Tranquilizer.class);
+ CuratorFramework c = mock(CuratorFramework.class);
+
+ @Override
+ public Tranquilizer getTranquilizer() {
+ return t;
+ }
+
+ @Override
+ CuratorFramework getCurator(String zkConnectString) {
+ return c;
+ }
+ }
+
+ public static class TestControllerServiceProcessor extends AbstractProcessor {
+
+ static final PropertyDescriptor CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Client Service")
+ .description("DruidTranquilityService")
+ .identifiesControllerService(DruidTranquilityService.class)
+ .required(true)
+ .build();
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ List<PropertyDescriptor> propertyDescriptors = new ArrayList<>();
+ propertyDescriptors.add(CLIENT_SERVICE);
+ return propertyDescriptors;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
new file mode 100644
index 0000000..6a384c6
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/pom.xml
@@ -0,0 +1,44 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-bundle</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-druid-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service-api-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-processors</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000..f3c8ece
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,209 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..b8c958c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-nar/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,5 @@
+nifi-druid-nar
+Copyright 2015-2017 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
new file mode 100644
index 0000000..f596c6f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/pom.xml
@@ -0,0 +1,74 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-bundle</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-druid-processors</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-service-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service-api</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.druid</groupId>
+ <artifactId>tranquility-core_2.11</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock-record-utils</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service</artifactId>
+ <version>1.5.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/java/org/apache/nifi/processors/druid/PutDruidRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/java/org/apache/nifi/processors/druid/PutDruidRecord.java b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/java/org/apache/nifi/processors/druid/PutDruidRecord.java
new file mode 100644
index 0000000..19e47a9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/java/org/apache/nifi/processors/druid/PutDruidRecord.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.druid;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.metamx.tranquility.tranquilizer.MessageDroppedException;
+import com.twitter.util.Future;
+import com.twitter.util.FutureEventListener;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+
+import org.apache.nifi.controller.api.druid.DruidTranquilityService;
+import com.metamx.tranquility.tranquilizer.Tranquilizer;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.util.DataTypeUtils;
+import scala.runtime.BoxedUnit;
+
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+@Tags({"druid", "timeseries", "olap", "ingest", "put", "record"})
+@CapabilityDescription("Sends records to Druid for Indexing. Leverages Druid Tranquility Controller service.")
+@WritesAttribute(attribute = "record.count", description = "The number of messages that were sent to Druid for this FlowFile. FlowFiles on the success relationship will have a value "
+ + "of this attribute that indicates the number of records successfully processed by Druid, and the FlowFile content will be only the successful records. This behavior applies "
+ + "to the failure and dropped relationships as well.")
+public class PutDruidRecord extends AbstractSessionFactoryProcessor {
+
+ static final String RECORD_COUNT = "record.count";
+
+ private List<PropertyDescriptor> properties;
+ private Set<Relationship> relationships;
+
+ static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("putdruid-record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("putdruid-record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use in order to serialize the data to outgoing relationships.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .expressionLanguageSupported(false)
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor DRUID_TRANQUILITY_SERVICE = new PropertyDescriptor.Builder()
+ .name("putdruid-tranquility-service")
+ .displayName("Tranquility Service")
+ .description("Tranquility Service to use for sending events to Druid.")
+ .required(true)
+ .identifiesControllerService(DruidTranquilityService.class)
+ .build();
+
+ static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("FlowFiles are routed to this relationship when they are successfully processed by Druid")
+ .build();
+
+ static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("FlowFiles are routed to this relationship when they cannot be parsed or otherwise processed by Druid")
+ .build();
+
+ static final Relationship REL_DROPPED = new Relationship.Builder()
+ .name("dropped")
+ .description("FlowFiles are routed to this relationship when they are outside of the configured time window, timestamp format is invalid, ect...")
+ .build();
+
+
+ public void init(final ProcessorInitializationContext context) {
+ List<PropertyDescriptor> properties = new ArrayList<>();
+ properties.add(RECORD_READER_FACTORY);
+ properties.add(RECORD_WRITER_FACTORY);
+ properties.add(DRUID_TRANQUILITY_SERVICE);
+ this.properties = Collections.unmodifiableList(properties);
+
+ Set<Relationship> relationships = new HashSet<>();
+ relationships.add(REL_SUCCESS);
+ relationships.add(REL_DROPPED);
+ relationships.add(REL_FAILURE);
+ this.relationships = Collections.unmodifiableSet(relationships);
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ /**
+ * Parses the record(s), converts each to a Map, and sends via Tranquility to the Druid Indexing Service
+ *
+ * @param context The process context
+ * @param session The process session
+ */
+ @SuppressWarnings("unchecked")
+ private void processFlowFile(ProcessContext context, final ProcessSession session) {
+ final ComponentLog log = getLogger();
+
+ // Get handle on Druid Tranquility session
+ DruidTranquilityService tranquilityController = context.getProperty(DRUID_TRANQUILITY_SERVICE)
+ .asControllerService(DruidTranquilityService.class);
+ Tranquilizer<Map<String, Object>> tranquilizer = tranquilityController.getTranquilizer();
+
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ // Create the outgoing flow files and output streams
+ FlowFile droppedFlowFile = session.create(flowFile);
+ final AtomicInteger droppedFlowFileCount = new AtomicInteger(0);
+ FlowFile failedFlowFile = session.create(flowFile);
+ final AtomicInteger failedFlowFileCount = new AtomicInteger(0);
+ FlowFile successfulFlowFile = session.create(flowFile);
+ final AtomicInteger successfulFlowFileCount = new AtomicInteger(0);
+
+ final AtomicInteger recordWriteErrors = new AtomicInteger(0);
+
+ int recordCount = 0;
+ final OutputStream droppedOutputStream = session.write(droppedFlowFile);
+ final RecordSetWriter droppedRecordWriter;
+ final OutputStream failedOutputStream = session.write(failedFlowFile);
+ final RecordSetWriter failedRecordWriter;
+ final OutputStream successfulOutputStream = session.write(successfulFlowFile);
+ final RecordSetWriter successfulRecordWriter;
+ try (final InputStream in = session.read(flowFile)) {
+
+ final RecordReaderFactory recordParserFactory = context.getProperty(RECORD_READER_FACTORY)
+ .asControllerService(RecordReaderFactory.class);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+
+ final Map<String, String> attributes = flowFile.getAttributes();
+
+ final RecordReader reader = recordParserFactory.createRecordReader(flowFile, in, getLogger());
+ final RecordSchema outSchema = writerFactory.getSchema(attributes, reader.getSchema());
+ droppedRecordWriter = writerFactory.createWriter(log, outSchema, droppedOutputStream);
+ droppedRecordWriter.beginRecordSet();
+ failedRecordWriter = writerFactory.createWriter(log, outSchema, failedOutputStream);
+ failedRecordWriter.beginRecordSet();
+ successfulRecordWriter = writerFactory.createWriter(log, outSchema, successfulOutputStream);
+ successfulRecordWriter.beginRecordSet();
+
+ Record r;
+ while ((r = reader.nextRecord()) != null) {
+ final Record record = r;
+ recordCount++;
+ // Convert each Record to HashMap and send to Druid
+ Map<String, Object> contentMap = (Map<String, Object>) DataTypeUtils.convertRecordFieldtoObject(r, RecordFieldType.RECORD.getRecordDataType(r.getSchema()));
+
+ log.debug("Tranquilizer Status: {}", new Object[]{tranquilizer.status().toString()});
+ // Send data element to Druid asynchronously
+ Future<BoxedUnit> future = tranquilizer.send(contentMap);
+ log.debug("Sent Payload to Druid: {}", new Object[]{contentMap});
+
+ // Wait for Druid to call back with status
+ future.addEventListener(new FutureEventListener<Object>() {
+ @Override
+ public void onFailure(Throwable cause) {
+ if (cause instanceof MessageDroppedException) {
+ // This happens when event timestamp targets a Druid Indexing task that has closed (Late Arriving Data)
+ log.debug("Record Dropped due to MessageDroppedException: {}, transferring record to dropped.", new Object[]{cause.getMessage()}, cause);
+ try {
+ synchronized (droppedRecordWriter) {
+ droppedRecordWriter.write(record);
+ droppedRecordWriter.flush();
+ droppedFlowFileCount.incrementAndGet();
+ }
+ } catch (final IOException ioe) {
+ log.error("Error transferring record to dropped, this may result in data loss.", new Object[]{ioe.getMessage()}, ioe);
+ recordWriteErrors.incrementAndGet();
+ }
+
+ } else {
+ log.error("FlowFile Processing Failed due to: {}", new Object[]{cause.getMessage()}, cause);
+ try {
+ synchronized (failedRecordWriter) {
+ failedRecordWriter.write(record);
+ failedRecordWriter.flush();
+ failedFlowFileCount.incrementAndGet();
+ }
+ } catch (final IOException ioe) {
+ log.error("Error transferring record to failure, this may result in data loss.", new Object[]{ioe.getMessage()}, ioe);
+ recordWriteErrors.incrementAndGet();
+ }
+ }
+ }
+
+ @Override
+ public void onSuccess(Object value) {
+ log.debug(" FlowFile Processing Success: {}", new Object[]{value.toString()});
+ try {
+ synchronized (successfulRecordWriter) {
+ successfulRecordWriter.write(record);
+ successfulRecordWriter.flush();
+ successfulFlowFileCount.incrementAndGet();
+ }
+ } catch (final IOException ioe) {
+ log.error("Error transferring record to success, this may result in data loss. "
+ + "However the record was successfully processed by Druid", new Object[]{ioe.getMessage()}, ioe);
+ recordWriteErrors.incrementAndGet();
+ }
+ }
+ });
+
+ }
+
+ } catch (IOException | SchemaNotFoundException | MalformedRecordException e) {
+ log.error("FlowFile Processing Failed due to: {}", new Object[]{e.getMessage()}, e);
+ // The FlowFile will be obtained and the error logged below, when calling publishResult.getFailedFlowFiles()
+ flowFile = session.putAttribute(flowFile, RECORD_COUNT, Integer.toString(recordCount));
+ session.transfer(flowFile, REL_FAILURE);
+ try {
+ droppedOutputStream.close();
+ session.remove(droppedFlowFile);
+ } catch (IOException ioe) {
+ log.error("Error closing output stream for FlowFile with dropped records.", ioe);
+ }
+ try {
+ failedOutputStream.close();
+ session.remove(failedFlowFile);
+ } catch (IOException ioe) {
+ log.error("Error closing output stream for FlowFile with failed records.", ioe);
+ }
+ try {
+ successfulOutputStream.close();
+ session.remove(successfulFlowFile);
+ } catch (IOException ioe) {
+ log.error("Error closing output stream for FlowFile with successful records.", ioe);
+ }
+ session.commit();
+ return;
+ }
+
+ if (recordCount == 0) {
+ // Send original (empty) flow file to success, remove the rest
+ flowFile = session.putAttribute(flowFile, RECORD_COUNT, "0");
+ session.transfer(flowFile, REL_SUCCESS);
+ try {
+ droppedOutputStream.close();
+ session.remove(droppedFlowFile);
+ } catch (IOException ioe) {
+ log.error("Error closing output stream for FlowFile with dropped records.", ioe);
+ }
+ try {
+ failedOutputStream.close();
+ session.remove(failedFlowFile);
+ } catch (IOException ioe) {
+ log.error("Error closing output stream for FlowFile with failed records.", ioe);
+ }
+ try {
+ successfulOutputStream.close();
+ session.remove(successfulFlowFile);
+ } catch (IOException ioe) {
+ log.error("Error closing output stream for FlowFile with successful records.", ioe);
+ }
+ } else {
+
+ // Wait for all the records to finish processing
+ while (recordCount != (droppedFlowFileCount.get() + failedFlowFileCount.get() + successfulFlowFileCount.get() + recordWriteErrors.get())) {
+ Thread.yield();
+ }
+
+ // Send partitioned flow files out to their relationships (or remove them if empty)
+
+ try {
+ droppedRecordWriter.finishRecordSet();
+ droppedRecordWriter.close();
+ } catch (IOException ioe) {
+ log.error("Error closing FlowFile with dropped records: {}", new Object[]{ioe.getMessage()}, ioe);
+ session.rollback();
+ throw new ProcessException(ioe);
+ }
+ if (droppedFlowFileCount.get() > 0) {
+ droppedFlowFile = session.putAttribute(droppedFlowFile, RECORD_COUNT, Integer.toString(droppedFlowFileCount.get()));
+ session.transfer(droppedFlowFile, REL_DROPPED);
+ } else {
+ session.remove(droppedFlowFile);
+ }
+
+ try {
+ failedRecordWriter.finishRecordSet();
+ failedRecordWriter.close();
+ } catch (IOException ioe) {
+ log.error("Error closing FlowFile with failed records: {}", new Object[]{ioe.getMessage()}, ioe);
+ session.rollback();
+ throw new ProcessException(ioe);
+ }
+ if (failedFlowFileCount.get() > 0) {
+ failedFlowFile = session.putAttribute(failedFlowFile, RECORD_COUNT, Integer.toString(failedFlowFileCount.get()));
+ session.transfer(failedFlowFile, REL_FAILURE);
+ } else {
+ session.remove(failedFlowFile);
+ }
+
+ try {
+ successfulRecordWriter.finishRecordSet();
+ successfulRecordWriter.close();
+ } catch (IOException ioe) {
+ log.error("Error closing FlowFile with successful records: {}", new Object[]{ioe.getMessage()}, ioe);
+ session.rollback();
+ throw new ProcessException(ioe);
+ }
+ if (successfulFlowFileCount.get() > 0) {
+ successfulFlowFile = session.putAttribute(successfulFlowFile, RECORD_COUNT, Integer.toString(successfulFlowFileCount.get()));
+ session.transfer(successfulFlowFile, REL_SUCCESS);
+ session.getProvenanceReporter().send(successfulFlowFile, tranquilityController.getTransitUri());
+ } else {
+ session.remove(successfulFlowFile);
+ }
+
+ session.remove(flowFile);
+ }
+
+ session.commit();
+ }
+
+ public void onTrigger(ProcessContext context, ProcessSessionFactory factory) throws ProcessException {
+ final ProcessSession session = factory.createSession();
+ processFlowFile(context, session);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
new file mode 100644
index 0000000..25ef747
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.nifi.processors.druid.PutDruidRecord
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/45edd12a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/controller/druid/MockDruidTranquilityController.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/controller/druid/MockDruidTranquilityController.java b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/controller/druid/MockDruidTranquilityController.java
new file mode 100644
index 0000000..695212c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-druid-bundle/nifi-druid-processors/src/test/java/org/apache/nifi/controller/druid/MockDruidTranquilityController.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.druid;
+
+import com.metamx.tranquility.beam.Beam;
+import com.metamx.tranquility.tranquilizer.MessageDroppedException;
+import com.metamx.tranquility.tranquilizer.Tranquilizer;
+import com.metamx.tranquility.typeclass.Timestamper;
+import com.twitter.finagle.Status;
+import com.twitter.util.Awaitable;
+import com.twitter.util.Duration;
+import com.twitter.util.Future;
+import com.twitter.util.FutureEventListener;
+import com.twitter.util.TimeoutException;
+import com.twitter.util.Try;
+import io.druid.data.input.impl.TimestampSpec;
+import io.druid.query.aggregation.AggregatorFactory;
+import org.apache.curator.framework.CuratorFramework;
+import scala.Function1;
+import scala.Option;
+import scala.runtime.BoxedUnit;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MockDruidTranquilityController extends DruidTranquilityController {
+
+ private final Tranquilizer t;
+ private final CuratorFramework cf;
+ private int numCalls = 0;
+
+ public MockDruidTranquilityController() {
+ this(-1, -1);
+ }
+
+ /**
+ * Creates a mock/stub Druid controller for testing. The failAfterN parameter must be higher than the dropAfterN parameter in order for messages to be dropped.
+ *
+ * @param dropAfterN The number of records after which to start calling the "dropped" callback, namely onFailure(MessageDroppedException)
+ * @param failAfterN The number of records after which to start calling the "failure" callback, namely onFailure(Exception)
+ */
+ public MockDruidTranquilityController(final int dropAfterN, final int failAfterN) {
+ t = mock(Tranquilizer.class);
+ final Future<BoxedUnit> future = new Future<BoxedUnit>() {
+
+ FutureEventListener<? super BoxedUnit> listener;
+
+ @Override
+ public Future<BoxedUnit> addEventListener(FutureEventListener<? super BoxedUnit> listener) {
+ this.listener = listener;
+ numCalls++;
+ if (dropAfterN >= 0 && numCalls > failAfterN) {
+ listener.onFailure(new Exception());
+ } else if (dropAfterN >= 0 && numCalls > dropAfterN) {
+ listener.onFailure(MessageDroppedException.Instance());
+ } else {
+ listener.onSuccess(BoxedUnit.UNIT);
+ }
+ return this;
+ }
+
+ @Override
+ public Awaitable<BoxedUnit> ready(Duration timeout, CanAwait permit) throws InterruptedException, TimeoutException {
+ return null;
+ }
+
+ @Override
+ public BoxedUnit result(Duration timeout, CanAwait permit) throws Exception {
+ return null;
+ }
+
+ @Override
+ public boolean isReady(CanAwait permit) {
+ return true;
+ }
+
+ @Override
+ public Future<BoxedUnit> respond(Function1<Try<BoxedUnit>, BoxedUnit> k) {
+ return null;
+ }
+
+ @Override
+ public Option<Try<BoxedUnit>> poll() {
+ return null;
+ }
+
+ @Override
+ public void raise(Throwable interrupt) {
+
+ }
+
+ @Override
+ public <B> Future<B> transform(Function1<Try<BoxedUnit>, Future<B>> f) {
+ return null;
+ }
+ };
+ when(t.send(anyObject())).thenReturn(future);
+ when(t.status()).thenReturn(new Status() {
+ });
+ cf = mock(CuratorFramework.class);
+ }
+
+ @Override
+ public Tranquilizer getTranquilizer() {
+ return t;
+ }
+
+ @Override
+ CuratorFramework getCurator(String zkConnectString) {
+ return cf;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ Tranquilizer<Map<String, Object>> buildTranquilizer(int maxBatchSize, int maxPendingBatches, int lingerMillis, Beam<Map<String, Object>> beam) {
+ return t;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ Beam<Map<String, Object>> buildBeam(String dataSource, String indexService, String discoveryPath, int clusterPartitions, int clusterReplication,
+ String segmentGranularity, String queryGranularity, String windowPeriod, String indexRetryPeriod, List<String> dimensions,
+ List<AggregatorFactory> aggregator, Timestamper<Map<String, Object>> timestamper, TimestampSpec timestampSpec) {
+ return mock(Beam.class);
+ }
+
+ @Override
+ public String getTransitUri() {
+ return "";
+ }
+
+}
[20/50] nifi git commit: BUG-86293: Set hive dependencies to last HDF
version
Posted by al...@apache.org.
BUG-86293: Set hive dependencies to last HDF version
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/73a4909c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/73a4909c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/73a4909c
Branch: refs/heads/HDF-3.1-maint
Commit: 73a4909cff44c9f2f659f83edc22ab2ea6757235
Parents: 9d7cdb6
Author: Matthew Burgess <ma...@apache.org>
Authored: Fri Jan 19 18:32:09 2018 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Fri Jan 19 18:32:09 2018 -0500
----------------------------------------------------------------------
pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/73a4909c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fc69e0b..b762c5c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,8 +103,8 @@
<hadoop.http.core.version>4.2.5</hadoop.http.core.version>
<yammer.metrics.version>2.2.0</yammer.metrics.version>
<ranger.version>0.7.1</ranger.version>
- <hive.version>1.2.1000.2.6.0.3-8</hive.version>
- <hive.hadoop.version>2.7.3.2.6.0.3-8</hive.hadoop.version>
+ <hive.version>1.2.1000.2.6.4.0-91</hive.version>
+ <hive.hadoop.version>2.7.3.2.6.4.0-91</hive.hadoop.version>
<hbase.version>1.1.2</hbase.version>
<storm.version>1.1.1</storm.version>
<hwx.registry.version>0.3.0</hwx.registry.version>
[24/50] nifi git commit: NIFI-4790 - support HTTPS Proxy in InvokeHTTP
Posted by al...@apache.org.
NIFI-4790 - support HTTPS Proxy in InvokeHTTP
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #2426.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4829f519
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4829f519
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4829f519
Branch: refs/heads/HDF-3.1-maint
Commit: 4829f519d2482e0e872a65cccadbc12217cdce36
Parents: b32e1a0
Author: Marco Gaido <ma...@gmail.com>
Authored: Fri Jan 19 11:02:56 2018 +0100
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 08:59:11 2018 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/InvokeHTTP.java | 40 ++++++++++++++++++--
1 file changed, 36 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/4829f519/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
index 300e9cb..5f6e66c 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
@@ -148,6 +148,9 @@ public final class InvokeHTTP extends AbstractProcessor {
EXCEPTION_CLASS, EXCEPTION_MESSAGE,
"uuid", "filename", "path")));
+ public static final String HTTP = "http";
+ public static final String HTTPS = "https";
+
// properties
public static final PropertyDescriptor PROP_METHOD = new PropertyDescriptor.Builder()
.name("HTTP Method")
@@ -213,11 +216,21 @@ public final class InvokeHTTP extends AbstractProcessor {
public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
.name("SSL Context Service")
- .description("The SSL Context Service used to provide client certificate information for TLS/SSL (https) connections.")
+ .description("The SSL Context Service used to provide client certificate information for TLS/SSL (https) connections."
+ + " It is also used to connect to HTTPS Proxy.")
.required(false)
.identifiesControllerService(SSLContextService.class)
.build();
+ public static final PropertyDescriptor PROP_PROXY_TYPE = new PropertyDescriptor.Builder()
+ .name("Proxy Type")
+ .displayName("Proxy Type")
+ .description("The type of the proxy we are connecting to. Must be either " + HTTP + " or " + HTTPS)
+ .defaultValue(HTTP)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
+ .build();
+
public static final PropertyDescriptor PROP_PROXY_HOST = new PropertyDescriptor.Builder()
.name("Proxy Host")
.description("The fully qualified hostname or IP address of the proxy server")
@@ -388,6 +401,7 @@ public final class InvokeHTTP extends AbstractProcessor {
PROP_BASIC_AUTH_PASSWORD,
PROP_PROXY_HOST,
PROP_PROXY_PORT,
+ PROP_PROXY_TYPE,
PROP_PROXY_USER,
PROP_PROXY_PASSWORD,
PROP_PUT_OUTPUT_IN_ATTRIBUTE,
@@ -509,10 +523,22 @@ public final class InvokeHTTP extends AbstractProcessor {
if ((proxyUserSet && !proxyPwdSet) || (!proxyUserSet && proxyPwdSet)) {
results.add(new ValidationResult.Builder().subject("Proxy User and Password").valid(false).explanation("If Proxy Username or Proxy Password is set, both must be set").build());
}
- if(proxyUserSet && !proxyHostSet) {
+ if (proxyUserSet && !proxyHostSet) {
results.add(new ValidationResult.Builder().subject("Proxy").valid(false).explanation("If Proxy username is set, proxy host must be set").build());
}
+ final String proxyType = validationContext.getProperty(PROP_PROXY_TYPE).evaluateAttributeExpressions().getValue();
+
+ if (!HTTP.equals(proxyType) && !HTTPS.equals(proxyType)) {
+ results.add(new ValidationResult.Builder().subject(PROP_PROXY_TYPE.getDisplayName()).valid(false)
+ .explanation(PROP_PROXY_TYPE.getDisplayName() + " must be either " + HTTP + " or " + HTTPS).build());
+ }
+
+ if (HTTPS.equals(proxyType)
+ && !validationContext.getProperty(PROP_SSL_CONTEXT_SERVICE).isSet()) {
+ results.add(new ValidationResult.Builder().subject("SSL Context Service").valid(false).explanation("If Proxy Type is HTTPS, SSL Context Service must be set").build());
+ }
+
return results;
}
@@ -525,9 +551,12 @@ public final class InvokeHTTP extends AbstractProcessor {
// Add a proxy if set
final String proxyHost = context.getProperty(PROP_PROXY_HOST).evaluateAttributeExpressions().getValue();
final Integer proxyPort = context.getProperty(PROP_PROXY_PORT).evaluateAttributeExpressions().asInteger();
+ final String proxyType = context.getProperty(PROP_PROXY_TYPE).evaluateAttributeExpressions().getValue();
+ boolean isHttpsProxy = false;
if (proxyHost != null && proxyPort != null) {
final Proxy proxy = new Proxy(Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
okHttpClientBuilder.proxy(proxy);
+ isHttpsProxy = HTTPS.equals(proxyType);
}
// Set timeouts
@@ -542,7 +571,7 @@ public final class InvokeHTTP extends AbstractProcessor {
// check if the ssl context is set and add the factory if so
if (sslContext != null) {
- setSslSocketFactory(okHttpClientBuilder, sslService, sslContext);
+ setSslSocketFactory(okHttpClientBuilder, sslService, sslContext, isHttpsProxy);
}
// check the trusted hostname property and override the HostnameVerifier
@@ -566,7 +595,7 @@ public final class InvokeHTTP extends AbstractProcessor {
In-depth documentation on Java Secure Socket Extension (JSSE) Classes and interfaces:
https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#JSSEClasses
*/
- private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, SSLContextService sslService, SSLContext sslContext)
+ private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, SSLContextService sslService, SSLContext sslContext, boolean setAsSocketFactory)
throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyManagementException {
final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
@@ -622,6 +651,9 @@ public final class InvokeHTTP extends AbstractProcessor {
final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();
okHttpClientBuilder.sslSocketFactory(sslSocketFactory, x509TrustManager);
+ if (setAsSocketFactory) {
+ okHttpClientBuilder.socketFactory(sslSocketFactory);
+ }
}
private void setAuthenticator(OkHttpClient.Builder okHttpClientBuilder, ProcessContext context) {
[21/50] nifi git commit: NIFI-4799: - Ensure variable registry update
request is marked completed. This closes #2421.
Posted by al...@apache.org.
NIFI-4799: - Ensure variable registry update request is marked completed. This closes #2421.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a261a0d8
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a261a0d8
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a261a0d8
Branch: refs/heads/HDF-3.1-maint
Commit: a261a0d86522f12d5cfcce02beec71bfdf3c11c3
Parents: 73a4909
Author: Matt Gilman <ma...@gmail.com>
Authored: Mon Jan 22 12:09:15 2018 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Jan 22 12:54:53 2018 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/web/api/ProcessGroupResource.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a261a0d8/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java
index 6e99a07..b9840f1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java
@@ -755,6 +755,9 @@ public class ProcessGroupResource extends ApplicationResource {
SecurityContextHolder.getContext().setAuthentication(authentication);
updateVariableRegistryReplicated(groupId, originalUri, activeAffectedProcessors, activeAffectedServices, updateRequest, requestVariableRegistryEntity);
+
+ // ensure the request is marked complete
+ updateRequest.setComplete(true);
} catch (final Exception e) {
logger.error("Failed to update variable registry", e);
[46/50] nifi git commit: NIFI-4773 - Fixed column type map
initialization in QueryDatabaseTable
Posted by al...@apache.org.
NIFI-4773 - Fixed column type map initialization in QueryDatabaseTable
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #2504.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b0142dcf
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b0142dcf
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b0142dcf
Branch: refs/heads/HDF-3.1-maint
Commit: b0142dcf987734956ba442f1c55ee1c32152c6fb
Parents: 1bed849
Author: Matthew Burgess <ma...@apache.org>
Authored: Fri Mar 2 11:19:43 2018 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Mon Apr 16 11:20:06 2018 -0400
----------------------------------------------------------------------
.../AbstractDatabaseFetchProcessor.java | 9 +-
.../processors/standard/QueryDatabaseTable.java | 7 ++
.../standard/QueryDatabaseTableTest.java | 79 ++++++++++++++++
.../standard/TestGenerateTableFetch.java | 96 ++++++++++++++++++++
4 files changed, 183 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b0142dcf/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
index 2145929..11a7685 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java
@@ -227,14 +227,6 @@ public abstract class AbstractDatabaseFetchProcessor extends AbstractSessionFact
return super.customValidate(validationContext);
}
- @Override
- public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
- // If the max-value columns have changed, we need to re-fetch the column info from the DB
- if (MAX_VALUE_COLUMN_NAMES.equals(descriptor) && newValue != null && !newValue.equals(oldValue)) {
- setupComplete.set(false);
- }
- }
-
public void setup(final ProcessContext context) {
setup(context,true,null);
}
@@ -246,6 +238,7 @@ public abstract class AbstractDatabaseFetchProcessor extends AbstractSessionFact
// If there are no max-value column names specified, we don't need to perform this processing
if (StringUtils.isEmpty(maxValueColumnNames)) {
+ setupComplete.set(true);
return;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b0142dcf/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
index ffe5845..2783d3c 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java
@@ -28,6 +28,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.state.Scope;
import org.apache.nifi.components.state.StateManager;
@@ -180,6 +181,12 @@ public class QueryDatabaseTable extends AbstractDatabaseFetchProcessor {
maxValueProperties = getDefaultMaxValueProperties(context.getProperties());
}
+ @OnStopped
+ public void stop() {
+ // Reset the column type map in case properties change
+ setupComplete.set(false);
+ }
+
@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
// Fetch the column/table info once
http://git-wip-us.apache.org/repos/asf/nifi/blob/b0142dcf/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java
index 8a2319b..9b0bc99 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java
@@ -308,6 +308,85 @@ public class QueryDatabaseTableTest {
}
@Test
+ public void testAddedRowsTwoTables() throws ClassNotFoundException, SQLException, InitializationException, IOException {
+
+ // load test data to database
+ final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection();
+ Statement stmt = con.createStatement();
+
+ try {
+ stmt.execute("drop table TEST_QUERY_DB_TABLE");
+ } catch (final SQLException sqle) {
+ // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842]
+ }
+
+ try {
+ stmt.execute("drop table TEST_QUERY_DB_TABLE2");
+ } catch (final SQLException sqle) {
+ // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842]
+ }
+
+ stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')");
+
+ runner.setProperty(QueryDatabaseTable.TABLE_NAME, "TEST_QUERY_DB_TABLE");
+ runner.setIncomingConnection(false);
+ runner.setProperty(QueryDatabaseTable.MAX_VALUE_COLUMN_NAMES, "ID");
+ runner.setProperty(QueryDatabaseTable.MAX_ROWS_PER_FLOW_FILE,"2");
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 2);
+
+ MockFlowFile flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0);
+ assertEquals("TEST_QUERY_DB_TABLE", flowFile.getAttribute(QueryDatabaseTable.RESULT_TABLENAME));
+ assertEquals(flowFile.getAttribute("maxvalue.id"), "2");
+ InputStream in = new ByteArrayInputStream(flowFile.toByteArray());
+ runner.setProperty(QueryDatabaseTable.FETCH_SIZE, "2");
+ assertEquals(2, getNumberOfRecordsFromStream(in));
+
+ flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(1);
+ assertEquals(flowFile.getAttribute("maxvalue.id"), "2");
+ in = new ByteArrayInputStream(flowFile.toByteArray());
+ assertEquals(1, getNumberOfRecordsFromStream(in));
+ runner.clearTransferState();
+
+ // Populate a second table and set
+ stmt.execute("create table TEST_QUERY_DB_TABLE2 (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')");
+
+ runner.setProperty(QueryDatabaseTable.TABLE_NAME, "TEST_QUERY_DB_TABLE2");
+ runner.setProperty(QueryDatabaseTable.MAX_ROWS_PER_FLOW_FILE,"0");
+ runner.run();
+ runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 1);
+
+ flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0);
+ assertEquals("TEST_QUERY_DB_TABLE2", flowFile.getAttribute(QueryDatabaseTable.RESULT_TABLENAME));
+ assertEquals(flowFile.getAttribute("maxvalue.id"), "2");
+ in = new ByteArrayInputStream(flowFile.toByteArray());
+ assertEquals(3, getNumberOfRecordsFromStream(in));
+ runner.clearTransferState();
+
+ // Add a new row with a higher ID and run, one flowfile with one new row should be transferred
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (3, 'Mary West', 15.0, '2000-01-01 03:23:34.234')");
+ runner.run();
+ runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 1);
+ flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0);
+ assertEquals(flowFile.getAttribute("maxvalue.id"), "3");
+ in = new ByteArrayInputStream(flowFile.toByteArray());
+ assertEquals(1, getNumberOfRecordsFromStream(in));
+
+ // Sanity check - run again, this time no flowfiles/rows should be transferred
+ runner.clearTransferState();
+ runner.run();
+ runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 0);
+ runner.clearTransferState();
+ }
+
+ @Test
public void testMultiplePartitions() throws ClassNotFoundException, SQLException, InitializationException, IOException {
// load test data to database
http://git-wip-us.apache.org/repos/asf/nifi/blob/b0142dcf/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
index 67a9bad..117b47e 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
@@ -219,6 +219,102 @@ public class TestGenerateTableFetch {
}
@Test
+ public void testAddedRowsTwoTables() throws ClassNotFoundException, SQLException, InitializationException, IOException {
+
+ // load test data to database
+ final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection();
+ Statement stmt = con.createStatement();
+
+ try {
+ stmt.execute("drop table TEST_QUERY_DB_TABLE");
+ } catch (final SQLException sqle) {
+ // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842]
+ }
+
+ try {
+ stmt.execute("drop table TEST_QUERY_DB_TABLE2");
+ } catch (final SQLException sqle) {
+ // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842]
+ }
+
+ stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')");
+
+ runner.setProperty(GenerateTableFetch.TABLE_NAME, "TEST_QUERY_DB_TABLE");
+ runner.setIncomingConnection(false);
+ runner.setProperty(GenerateTableFetch.MAX_VALUE_COLUMN_NAMES, "ID");
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
+ MockFlowFile flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0);
+ String query = new String(flowFile.toByteArray());
+ assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE WHERE ID <= 2 ORDER BY ID FETCH NEXT 10000 ROWS ONLY", query);
+ ResultSet resultSet = stmt.executeQuery(query);
+ // Should be three records
+ assertTrue(resultSet.next());
+ assertTrue(resultSet.next());
+ assertTrue(resultSet.next());
+ assertFalse(resultSet.next());
+ runner.clearTransferState();
+
+ // Run again, this time no flowfiles/rows should be transferred
+ runner.run();
+ runner.assertAllFlowFilesTransferred(REL_SUCCESS, 0);
+ runner.clearTransferState();
+
+ // Create and populate a new table and re-run
+ stmt.execute("create table TEST_QUERY_DB_TABLE2 (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')");
+
+ runner.setProperty(GenerateTableFetch.TABLE_NAME, "TEST_QUERY_DB_TABLE2");
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
+ flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0);
+ query = new String(flowFile.toByteArray());
+ assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE2 WHERE ID <= 2 ORDER BY ID FETCH NEXT 10000 ROWS ONLY", query);
+ resultSet = stmt.executeQuery(query);
+ // Should be three records
+ assertTrue(resultSet.next());
+ assertTrue(resultSet.next());
+ assertTrue(resultSet.next());
+ assertFalse(resultSet.next());
+ runner.clearTransferState();
+
+ // Add 3 new rows with a higher ID and run with a partition size of 2. Two flow files should be transferred
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (3, 'Mary West', 15.0, '2000-01-01 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (4, 'Marty Johnson', 15.0, '2011-01-01 03:23:34.234')");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (5, 'Marty Johnson', 15.0, '2011-01-01 03:23:34.234')");
+ runner.setProperty(GenerateTableFetch.PARTITION_SIZE, "2");
+ runner.run();
+ runner.assertAllFlowFilesTransferred(REL_SUCCESS, 2);
+
+ // Verify first flow file's contents
+ flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0);
+ query = new String(flowFile.toByteArray());
+ assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE2 WHERE ID > 2 AND ID <= 5 ORDER BY ID FETCH NEXT 2 ROWS ONLY", query);
+ resultSet = stmt.executeQuery(query);
+ // Should be two records
+ assertTrue(resultSet.next());
+ assertTrue(resultSet.next());
+ assertFalse(resultSet.next());
+
+ // Verify second flow file's contents
+ flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(1);
+ query = new String(flowFile.toByteArray());
+ assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE2 WHERE ID > 2 AND ID <= 5 ORDER BY ID OFFSET 2 ROWS FETCH NEXT 2 ROWS ONLY", query);
+ resultSet = stmt.executeQuery(query);
+ // Should be one record
+ assertTrue(resultSet.next());
+ assertFalse(resultSet.next());
+ runner.clearTransferState();
+ }
+
+ @Test
public void testAddedRowsRightBounded() throws ClassNotFoundException, SQLException, InitializationException, IOException {
// load test data to database
[27/50] nifi git commit: NIFI-4848: Update HttpComponents version
Posted by al...@apache.org.
NIFI-4848: Update HttpComponents version
- httpclient 4.5.3 -> 4.5.5
- httpcore 4.4.4 -> 4.4.9
- ThreadSafe annotation is removed since 4.4.5, HTTPCLIENT-1743.
Removed the annotation from DebugFlow processor.
- httpasyncclient 4.1.2 -> 4.1.3
- This closes #2453
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/678dfc1a
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/678dfc1a
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/678dfc1a
Branch: refs/heads/HDF-3.1-maint
Commit: 678dfc1aefb919dc290fc0ed8e2169b0c2556b2c
Parents: a0bd611
Author: Koji Kawamura <ij...@apache.org>
Authored: Wed Feb 7 15:42:54 2018 +0900
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 09:02:38 2018 -0500
----------------------------------------------------------------------
nifi-commons/nifi-site-to-site-client/pom.xml | 2 +-
.../main/java/org/apache/nifi/processors/standard/DebugFlow.java | 2 --
pom.xml | 4 ++--
3 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/678dfc1a/nifi-commons/nifi-site-to-site-client/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/pom.xml b/nifi-commons/nifi-site-to-site-client/pom.xml
index 21e8fb2..08a33db 100644
--- a/nifi-commons/nifi-site-to-site-client/pom.xml
+++ b/nifi-commons/nifi-site-to-site-client/pom.xml
@@ -62,7 +62,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
- <version>4.1.2</version>
+ <version>4.1.3</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/nifi/blob/678dfc1a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DebugFlow.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DebugFlow.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DebugFlow.java
index 3d12ee5..6e55157 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DebugFlow.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DebugFlow.java
@@ -28,7 +28,6 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.http.annotation.ThreadSafe;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
@@ -51,7 +50,6 @@ import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
-@ThreadSafe()
@EventDriven()
@Tags({"test", "debug", "processor", "utility", "flow", "FlowFile"})
@CapabilityDescription("The DebugFlow processor aids testing and debugging the FlowFile framework by allowing various "
http://git-wip-us.apache.org/repos/asf/nifi/blob/678dfc1a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3b2f1b2..3d800ce 100644
--- a/pom.xml
+++ b/pom.xml
@@ -308,12 +308,12 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.5.3</version>
+ <version>4.5.5</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.4.4</version>
+ <version>4.4.9</version>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
[26/50] nifi git commit: NIFI-4818: Fix transit URL parsing at
Hive2JDBC and KafkaTopic for ReportLineageToAtlas
Posted by al...@apache.org.
NIFI-4818: Fix transit URL parsing at Hive2JDBC and KafkaTopic for ReportLineageToAtlas
- Hive2JDBC: Handle connection parameters and multiple host entries
correctly
- KafkaTopic: Handle multiple host entries correctly
- Avoid potential "IllegalStateException: Duplicate key" exception
when NiFiAtlasHook analyzes existing NiFiFlowPath input/output entries
- This closes #2435
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a0bd6113
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a0bd6113
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a0bd6113
Branch: refs/heads/HDF-3.1-maint
Commit: a0bd61138a095217d3092e3bb29ab6971d019db0
Parents: 0ddd0b1
Author: Koji Kawamura <ij...@apache.org>
Authored: Thu Jan 25 13:57:01 2018 +0900
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 09:01:41 2018 -0500
----------------------------------------------------------------------
.../org/apache/nifi/atlas/NiFiAtlasHook.java | 6 +-
.../atlas/provenance/analyzer/Hive2JDBC.java | 46 +++++--
.../atlas/provenance/analyzer/KafkaTopic.java | 10 +-
.../provenance/analyzer/TestHive2JDBC.java | 133 +++++++++++++++++++
.../provenance/analyzer/TestKafkaTopic.java | 3 +-
5 files changed, 179 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a0bd6113/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/NiFiAtlasHook.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/NiFiAtlasHook.java b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/NiFiAtlasHook.java
index a15c935..58945d5 100644
--- a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/NiFiAtlasHook.java
+++ b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/NiFiAtlasHook.java
@@ -255,7 +255,11 @@ public class NiFiAtlasHook extends AtlasHook implements LineageContext {
}
return new Tuple<>(refQualifiedName, typedQualifiedNameToRef.get(toTypedQualifiedName(typeName, refQualifiedName)));
}).filter(Objects::nonNull).filter(tuple -> tuple.getValue() != null)
- .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue));
+ // If duplication happens, use new value.
+ .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue, (oldValue, newValue) -> {
+ logger.warn("Duplicated qualified name was found, use the new one. oldValue={}, newValue={}", new Object[]{oldValue, newValue});
+ return newValue;
+ }));
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/nifi/blob/a0bd6113/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/Hive2JDBC.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/Hive2JDBC.java b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/Hive2JDBC.java
index e393a09..ccbbc66 100644
--- a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/Hive2JDBC.java
+++ b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/Hive2JDBC.java
@@ -21,10 +21,14 @@ import org.apache.nifi.atlas.provenance.AnalysisContext;
import org.apache.nifi.atlas.provenance.DataSetRefs;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.util.StringUtils;
import org.apache.nifi.util.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static org.apache.nifi.atlas.provenance.analyzer.DatabaseAnalyzerUtil.ATTR_INPUT_TABLES;
import static org.apache.nifi.atlas.provenance.analyzer.DatabaseAnalyzerUtil.ATTR_OUTPUT_TABLES;
@@ -49,17 +53,41 @@ import static org.apache.nifi.atlas.provenance.analyzer.DatabaseAnalyzerUtil.par
*/
public class Hive2JDBC extends AbstractHiveAnalyzer {
+ private static final Logger logger = LoggerFactory.getLogger(Hive2JDBC.class);
+
+ // jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
+ // Group 1 = <host1>:<port1>,<host2>:<port2>
+ // Group 2 = dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
+ private static final String URI_PATTERN_STR = "jdbc:hive2://([^/]+)/?(.*)$";
+ private static final Pattern URI_PATTERN = Pattern.compile(URI_PATTERN_STR);
+
@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
// Replace the colon so that the schema in the URI can be parsed correctly.
- final String transitUri = event.getTransitUri().replaceFirst("^jdbc:hive2", "jdbc-hive2");
- final URI uri = parseUri(transitUri);
- final String clusterName = context.getClusterResolver().fromHostNames(uri.getHost());
- // Remove the heading '/'
- final String path = uri.getPath();
- // If uri does not contain database name, then use 'default' as database name.
- final String connectedDatabaseName = path == null || path.isEmpty() ? "default" : path.substring(1);
+ final String transitUri = event.getTransitUri();
+ if (transitUri == null) {
+ return null;
+ }
+
+ final Matcher uriMatcher = URI_PATTERN.matcher(transitUri);
+ if (!uriMatcher.matches()) {
+ logger.warn("Unexpected transit URI: {}", new Object[]{transitUri});
+ return null;
+ }
+
+ final String clusterName = context.getClusterResolver().fromHostNames(splitHostNames(uriMatcher.group(1)));
+ String connectedDatabaseName = null;
+ if (uriMatcher.groupCount() > 1) {
+ // Try to find connected database name from connection parameters.
+ final String[] connectionParams = uriMatcher.group(2).split(";");
+ connectedDatabaseName = connectionParams[0];
+ }
+
+ if (StringUtils.isEmpty(connectedDatabaseName)) {
+ // If not found, then use "default".
+ connectedDatabaseName = "default";
+ }
final Set<Tuple<String, String>> inputTables = parseTableNames(connectedDatabaseName, event.getAttribute(ATTR_INPUT_TABLES));
final Set<Tuple<String, String>> outputTables = parseTableNames(connectedDatabaseName, event.getAttribute(ATTR_OUTPUT_TABLES));
@@ -97,6 +125,6 @@ public class Hive2JDBC extends AbstractHiveAnalyzer {
@Override
public String targetTransitUriPattern() {
- return "^jdbc:hive2://.+$";
+ return URI_PATTERN_STR;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a0bd6113/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/KafkaTopic.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/KafkaTopic.java b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/KafkaTopic.java
index e3d4709..ff86166 100644
--- a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/KafkaTopic.java
+++ b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/main/java/org/apache/nifi/atlas/provenance/analyzer/KafkaTopic.java
@@ -62,14 +62,8 @@ public class KafkaTopic extends AbstractNiFiProvenanceEventAnalyzer {
return null;
}
- String clusterName = null;
- for (String broker : uriMatcher.group(1).split(",")) {
- final String brokerHostname = broker.split(":")[0].trim();
- clusterName = context.getClusterResolver().fromHostNames(brokerHostname);
- if (clusterName != null && !clusterName.isEmpty()) {
- break;
- }
- }
+ final String[] hostNames = splitHostNames(uriMatcher.group(1));
+ final String clusterName = context.getClusterResolver().fromHostNames(hostNames);
final String topicName = uriMatcher.group(2);
http://git-wip-us.apache.org/repos/asf/nifi/blob/a0bd6113/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestHive2JDBC.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestHive2JDBC.java b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestHive2JDBC.java
index 9e1a92c..f63d8ca 100644
--- a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestHive2JDBC.java
+++ b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestHive2JDBC.java
@@ -37,6 +37,7 @@ import static org.apache.nifi.atlas.provenance.analyzer.DatabaseAnalyzerUtil.ATT
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.matches;
import static org.mockito.Mockito.when;
@@ -160,4 +161,136 @@ public class TestHive2JDBC {
assertEquals("tableB1", ref.get(ATTR_NAME));
assertEquals("databaseB.tableB1@cluster1", ref.get(ATTR_QUALIFIED_NAME));
}
+
+ /**
+ * A Hive connection URL can have connection strings delimited by semicolons.
+ */
+ @Test
+ public void testTableLineageWithDefaultTableNameWithConnectionParams() {
+ final String processorName = "PutHiveQL";
+ final String transitUri = "jdbc:hive2://0.example.com:10000;transportMode=http;httpPath=cliservice";
+ final ProvenanceEventRecord record = Mockito.mock(ProvenanceEventRecord.class);
+ when(record.getComponentType()).thenReturn(processorName);
+ when(record.getTransitUri()).thenReturn(transitUri);
+ when(record.getEventType()).thenReturn(ProvenanceEventType.SEND);
+ // E.g. insert into databaseB.tableB1 select something from tableA1 a1 inner join tableA2 a2 where a1.id = a2.id
+ when(record.getAttribute(ATTR_INPUT_TABLES)).thenReturn("tableA1, tableA2");
+ when(record.getAttribute(ATTR_OUTPUT_TABLES)).thenReturn("databaseB.tableB1");
+
+ final ClusterResolvers clusterResolvers = Mockito.mock(ClusterResolvers.class);
+ when(clusterResolvers.fromHostNames(matches(".+\\.example\\.com"))).thenReturn("cluster1");
+
+ final AnalysisContext context = Mockito.mock(AnalysisContext.class);
+ when(context.getClusterResolver()).thenReturn(clusterResolvers);
+
+ final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(processorName, transitUri, record.getEventType());
+ assertNotNull(analyzer);
+
+ final DataSetRefs refs = analyzer.analyze(context, record);
+ assertEquals(2, refs.getInputs().size());
+ // QualifiedName : Name
+ final Map<String, String> expectedInputRefs = new HashMap<>();
+ expectedInputRefs.put("default.tableA1@cluster1", "tableA1");
+ expectedInputRefs.put("default.tableA2@cluster1", "tableA2");
+ for (Referenceable ref : refs.getInputs()) {
+ final String qName = (String) ref.get(ATTR_QUALIFIED_NAME);
+ assertTrue(expectedInputRefs.containsKey(qName));
+ assertEquals(expectedInputRefs.get(qName), ref.get(ATTR_NAME));
+ }
+
+ assertEquals(1, refs.getOutputs().size());
+ Referenceable ref = refs.getOutputs().iterator().next();
+ assertEquals("hive_table", ref.getTypeName());
+ assertEquals("tableB1", ref.get(ATTR_NAME));
+ assertEquals("databaseB.tableB1@cluster1", ref.get(ATTR_QUALIFIED_NAME));
+ }
+
+ /**
+ * Hive connection URL can have multiple zookeeper host ports
+ * and multiple parameters delimited with semicolons.
+ * Database name can be omitted.
+ */
+ @Test
+ public void testTableLineageWithZookeeperDiscovery() {
+ final String processorName = "PutHiveQL";
+ final String transitUri = "jdbc:hive2://0.example.com:2181,1.example.com:2181,2.example.com:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2";
+ final ProvenanceEventRecord record = Mockito.mock(ProvenanceEventRecord.class);
+ when(record.getComponentType()).thenReturn(processorName);
+ when(record.getTransitUri()).thenReturn(transitUri);
+ when(record.getEventType()).thenReturn(ProvenanceEventType.SEND);
+ // E.g. insert into databaseB.tableB1 select something from tableA1 a1 inner join tableA2 a2 where a1.id = a2.id
+ when(record.getAttribute(ATTR_INPUT_TABLES)).thenReturn("tableA1, tableA2");
+ when(record.getAttribute(ATTR_OUTPUT_TABLES)).thenReturn("databaseB.tableB1");
+
+ final ClusterResolvers clusterResolvers = Mockito.mock(ClusterResolvers.class);
+ when(clusterResolvers.fromHostNames(eq("0.example.com"), eq("1.example.com"), eq("2.example.com"))).thenReturn("cluster1");
+
+ final AnalysisContext context = Mockito.mock(AnalysisContext.class);
+ when(context.getClusterResolver()).thenReturn(clusterResolvers);
+
+ final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(processorName, transitUri, record.getEventType());
+ assertNotNull(analyzer);
+
+ final DataSetRefs refs = analyzer.analyze(context, record);
+ assertEquals(2, refs.getInputs().size());
+ // QualifiedName : Name
+ final Map<String, String> expectedInputRefs = new HashMap<>();
+ expectedInputRefs.put("default.tableA1@cluster1", "tableA1");
+ expectedInputRefs.put("default.tableA2@cluster1", "tableA2");
+ for (Referenceable ref : refs.getInputs()) {
+ final String qName = (String) ref.get(ATTR_QUALIFIED_NAME);
+ assertTrue(expectedInputRefs.containsKey(qName));
+ assertEquals(expectedInputRefs.get(qName), ref.get(ATTR_NAME));
+ }
+
+ assertEquals(1, refs.getOutputs().size());
+ Referenceable ref = refs.getOutputs().iterator().next();
+ assertEquals("hive_table", ref.getTypeName());
+ assertEquals("tableB1", ref.get(ATTR_NAME));
+ assertEquals("databaseB.tableB1@cluster1", ref.get(ATTR_QUALIFIED_NAME));
+ }
+
+ /**
+ * Hive connection URL using zookeeper and database name.
+ */
+ @Test
+ public void testTableLineageWithZookeeperDiscoverySpecificDatabase() {
+ final String processorName = "PutHiveQL";
+ final String transitUri = "jdbc:hive2://0.example.com:2181,1.example.com:2181/some_database;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2";
+ final ProvenanceEventRecord record = Mockito.mock(ProvenanceEventRecord.class);
+ when(record.getComponentType()).thenReturn(processorName);
+ when(record.getTransitUri()).thenReturn(transitUri);
+ when(record.getEventType()).thenReturn(ProvenanceEventType.SEND);
+ // E.g. insert into databaseB.tableB1 select something from tableA1 a1 inner join tableA2 a2 where a1.id = a2.id
+ when(record.getAttribute(ATTR_INPUT_TABLES)).thenReturn("tableA1, tableA2");
+ when(record.getAttribute(ATTR_OUTPUT_TABLES)).thenReturn("databaseB.tableB1");
+
+ final ClusterResolvers clusterResolvers = Mockito.mock(ClusterResolvers.class);
+ when(clusterResolvers.fromHostNames(eq("0.example.com"), eq("1.example.com"))).thenReturn("cluster1");
+
+ final AnalysisContext context = Mockito.mock(AnalysisContext.class);
+ when(context.getClusterResolver()).thenReturn(clusterResolvers);
+
+ final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(processorName, transitUri, record.getEventType());
+ assertNotNull(analyzer);
+
+ final DataSetRefs refs = analyzer.analyze(context, record);
+ assertEquals(2, refs.getInputs().size());
+ // QualifiedName : Name
+ final Map<String, String> expectedInputRefs = new HashMap<>();
+ expectedInputRefs.put("some_database.tableA1@cluster1", "tableA1");
+ expectedInputRefs.put("some_database.tableA2@cluster1", "tableA2");
+ for (Referenceable ref : refs.getInputs()) {
+ final String qName = (String) ref.get(ATTR_QUALIFIED_NAME);
+ assertTrue(expectedInputRefs.containsKey(qName));
+ assertEquals(expectedInputRefs.get(qName), ref.get(ATTR_NAME));
+ }
+
+ assertEquals(1, refs.getOutputs().size());
+ Referenceable ref = refs.getOutputs().iterator().next();
+ assertEquals("hive_table", ref.getTypeName());
+ assertEquals("tableB1", ref.get(ATTR_NAME));
+ assertEquals("databaseB.tableB1@cluster1", ref.get(ATTR_QUALIFIED_NAME));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a0bd6113/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestKafkaTopic.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestKafkaTopic.java b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestKafkaTopic.java
index 5c0fd0e..543ac89 100644
--- a/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestKafkaTopic.java
+++ b/nifi-nar-bundles/nifi-atlas-bundle/nifi-atlas-reporting-task/src/test/java/org/apache/nifi/atlas/provenance/analyzer/TestKafkaTopic.java
@@ -31,6 +31,7 @@ import static org.apache.nifi.atlas.NiFiTypes.ATTR_NAME;
import static org.apache.nifi.atlas.NiFiTypes.ATTR_QUALIFIED_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.matches;
import static org.mockito.Mockito.when;
@@ -73,7 +74,7 @@ public class TestKafkaTopic {
when(record.getEventType()).thenReturn(ProvenanceEventType.SEND);
final ClusterResolvers clusterResolvers = Mockito.mock(ClusterResolvers.class);
- when(clusterResolvers.fromHostNames(matches(".+\\.example\\.com"))).thenReturn("cluster1");
+ when(clusterResolvers.fromHostNames(eq("0.example.com"), eq("1.example.com"))).thenReturn("cluster1");
final AnalysisContext context = Mockito.mock(AnalysisContext.class);
when(context.getClusterResolver()).thenReturn(clusterResolvers);
[14/50] nifi git commit: apply hortonworks branding
Posted by al...@apache.org.
apply hortonworks branding
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d3098192
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d3098192
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d3098192
Branch: refs/heads/HDF-3.1-maint
Commit: d30981924b2a74cda1c99d37ac066a5b8a049ae3
Parents: 4d14368
Author: Scott Aslan <sc...@gmail.com>
Authored: Tue May 9 14:52:10 2017 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Jan 16 15:30:28 2018 -0500
----------------------------------------------------------------------
.../WEB-INF/partials/canvas/about-dialog.jsp | 12 +++--
.../WEB-INF/partials/canvas/canvas-header.jsp | 2 +-
.../nifi-web-ui/src/main/webapp/css/about.css | 54 ++++++++++++++++++--
.../nifi-web-ui/src/main/webapp/css/header.css | 4 +-
.../src/main/webapp/css/navigation.css | 2 +-
.../src/main/webapp/images/hdf-footer.svg | 17 ++++++
.../src/main/webapp/images/hdf-header.svg | 17 ++++++
.../src/main/webapp/images/hdf-icon.svg | 17 ++++++
.../src/main/webapp/images/hwx-logo.svg | 17 ++++++
.../src/main/webapp/images/nifi-drop.svg | 2 +-
.../nf-ng-canvas-global-menu-controller.js | 26 +++++++---
.../views/nf-ng-breadcrumbs-directive-view.html | 1 +
12 files changed, 149 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/about-dialog.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/about-dialog.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/about-dialog.jsp
index 6453da6..8c4736d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/about-dialog.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/about-dialog.jsp
@@ -17,11 +17,13 @@
<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
<div id="nf-about">
<div id="nf-about-pic-container">
+ <img id="nf-about-hdf-icon" src="images/hdf-icon.svg">
+ <p id="nf-about-hdf-version-number" title="Version 3.1.0">Version 3.1.0</p>
<div id="nf-about-pic"></div>
</div>
<div class="dialog-content">
<div id="nf-about-content">
- <span id="nf-version"></span>
+ <p id="nf-version"></p>
<div id="nf-version-detail">
<p id="nf-version-detail-timestamp">
<span id="nf-about-build-timestamp"></span>
@@ -33,12 +35,14 @@
From <span id="nf-about-build-revision"></span> on branch <span id="nf-about-build-branch"></span>
</p>
</div>
- <p>
+ <div id="nf-about-description-container">
+ <img id="nf-about-nifi-drop" src="images/nifi-drop.svg">
+ <p id="nf-about-description">
Apache NiFi is a framework to support highly scalable and flexible dataflows.
It can be run on laptops up through clusters of enterprise class servers.
Instead of dictating a particular dataflow or behavior it empowers you to design your own
- optimal dataflow tailored to your specific environment.
- </p>
+ optimal dataflow tailored to your specific environment.</p>
+ </div>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
index 7dd481f..79415e8 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
@@ -16,7 +16,7 @@
--%>
<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
<md-toolbar id="header" layout-align="space-between center" layout="row" class="md-small md-accent md-hue-1">
- <img id="nifi-logo" src="images/nifi-logo.svg">
+ <img id="nifi-logo" src="images/hdf-header.svg">
<div flex layout="row" layout-align="space-between center">
<div id="component-container">
<button title="{{appCtrl.serviceProvider.headerCtrl.toolboxCtrl.config.type.processor}}"
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/about.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/about.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/about.css
index 0c29af2..e815c49 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/about.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/about.css
@@ -17,13 +17,17 @@
#nf-about {
overflow: hidden;
width: 500px;
- height: 500px;
- max-height: 450px;
+ height: 543px;
+ max-height: 543px;
+}
+
+#nf-about .dialog-header {
+ background-color: #3fae2a;
}
#nf-about-pic {
height: 60%;
- background: url(../images/nifi-logo-about.svg) #000000 no-repeat center center;
+ background: url(../images/hwx-logo.svg) #000000 no-repeat center center;
top: 20%;
position: relative;
}
@@ -37,13 +41,17 @@
font-weight:500;
font-size: 13px;
cursor: pointer;
+ padding: 0px;
}
#nf-about-content p {
- padding-top: 9px;
+ padding-top: 15px;
padding-bottom: 9px;
font-size: 15px;
line-height: 15px;
+ text-overflow: ellipsis;
+ overflow: hidden;
+ white-space: nowrap;
}
#nf-about-content span.link {
@@ -52,8 +60,44 @@
font-weight:900;
}
+#nf-about-description {
+ margin-left: 68px;
+ white-space: normal !important;
+}
+
+#nf-about-description-container {
+ position: relative;
+}
+
+#nf-about-nifi-drop {
+ position: absolute;
+ top: 15px;
+ left: 0px;
+ width: 48px;
+}
+
+#nf-about-hdf-icon {
+ position: absolute;
+ top: 65px;
+ left: 20px;
+ width: 48px;
+ z-index: 1;
+}
+
+#nf-about-hdf-version-number {
+ position: absolute;
+ top: 85px;
+ left: 77px;
+ z-index: 1;
+ color: #fff;
+ font-size: 13px;
+ text-overflow: ellipsis;
+ overflow: hidden;
+ white-space: nowrap;
+ right: 5px;
+}
+
#nf-version-detail {
- padding: 5px 0 0 5px;
display: none;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
index 95342bf..3f74e27 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
@@ -63,8 +63,8 @@ md-toolbar.md-small .md-toolbar-tools {
}
#nifi-logo {
- height: 37px;
- margin:0 20px 3px 15px;
+ height: 100%;
+ margin: 0 39px 0px 0px;
}
#global-menu-button {
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css
index 17200d7..7307b59 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css
@@ -299,7 +299,7 @@ span.breadcrumb-version-control-gray {
position: absolute;
left: 5px;
top: 8px;
- right: 5px;
+ right: 138px;
z-index: 4;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-footer.svg
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-footer.svg b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-footer.svg
new file mode 100644
index 0000000..f7d06e5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-footer.svg
@@ -0,0 +1,17 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 89 24"><defs><style>.cls-1{fill:#484848;}.cls-2{fill:#728d9b;}</style></defs><title>hdf-branding-graphics</title><g id="text-2" data-name="text"><path id="powered-by" class="cls-1" d="M2.56,8.72v3.06h-1V4H4.07a2.26,2.26,0,0,1,1.71.65,2.41,2.41,0,0,1,.62,1.74,2.39,2.39,0,0,1-.59,1.74,2.3,2.3,0,0,1-1.67.63Zm0-.84H4.07a1.27,1.27,0,0,0,1-.39,1.64,1.64,0,0,0,.35-1.12,1.69,1.69,0,0,0-.36-1.13,1.23,1.23,0,0,0-1-.42H2.56Zm11.08.68A3.9,3.9,0,0,1,13,11a2.33,2.33,0,0,1-1.94.87A2.35,2.35,0,0,1,9.11,11a3.71,3.71,0,0,1-.73-2.38V7.2a3.91,3.91,0,0,1,.69-2.46A2.31,2.31,0,0,1,11,3.86a2.34,2.34,0,0,1,1.93.85,3.83,3.83,0,0,1,.71,2.43Zm-1-1.37a3.35,3.35,0,0,0-.41-1.85A1.39,1.39,0,0,0,11,4.74a1.39,1.39,0,0,0-1.23.61,3.25,3.25,0,0,0-.42,1.8v1.4a3.31,3.31,0,0,0,.42,1.83A1.39,1.39,0,0,0,11,11a1.38,1.38,0,0,0,1.22-.58,3.27,3.27,0,0,0,.41-1.78Zm4.88,2.13.1.88.16-.77L19,4h.82l1.21,5.46.16.79.11-.9L22.3,4h1l-1.56,7.82h-.89l-1.3-5.7-.08-.46-.08.46L18,11.78h-.8
9L15.58,4h1ZM29,8.17H26.28v2.77h3.22v.84H25.3V4h4.15v.84H26.28V7.32H29Zm5,.45H32.63v3.16h-1V4h2.19a2.34,2.34,0,0,1,1.74.6,2.4,2.4,0,0,1,.59,1.75,2.44,2.44,0,0,1-.32,1.26,2,2,0,0,1-.91.81l1.52,3.32v.06H35.4Zm-1.37-.84h1.19a1.28,1.28,0,0,0,1-.4,1.51,1.51,0,0,0,.37-1.07q0-1.51-1.36-1.51H32.63Zm9.48.39H39.34v2.77h3.22v.84H38.36V4h4.15v.84H39.34V7.32h2.77Zm2.59,3.62V4h1.86a2.69,2.69,0,0,1,2.13.86,3.52,3.52,0,0,1,.76,2.41V8.54a3.45,3.45,0,0,1-.77,2.4,2.86,2.86,0,0,1-2.23.85Zm1-7v6.13h.79A1.83,1.83,0,0,0,48,10.35a2.8,2.8,0,0,0,.49-1.76V7.2A2.83,2.83,0,0,0,48,5.38a1.75,1.75,0,0,0-1.44-.58Zm9.58,7V4h2.23a2.23,2.23,0,0,1,1.61.52A2.07,2.07,0,0,1,59.64,6a1.73,1.73,0,0,1-.27,1,1.85,1.85,0,0,1-.74.65,1.48,1.48,0,0,1,.85.66,2.21,2.21,0,0,1,.31,1.2,2.29,2.29,0,0,1-.58,1.65,2.17,2.17,0,0,1-1.65.6Zm1-4.48h1.26A1.07,1.07,0,0,0,58.33,7,1.31,1.31,0,0,0,58.65,6a1.31,1.31,0,0,0-.28-.94,1.16,1.16,0,0,0-.87-.29H56.25Zm0,.83v2.81h1.34a1.13,1.13,0,0,0,.89-.37,1.5,1.5,0,0,0,.33-1q0-1.41-1.19-1.41Zm7.64-.24L65.
48,4h1.11l-2.22,4.9v2.92h-1V8.87L61.16,4h1.12Z"/><path id="apache-nifi" class="cls-1" d="M5.13,20.18H3l-.42,1.6H.89L3.33,14H4.78l2.46,7.82H5.55ZM3.32,18.87H4.78l-.73-2.79Zm7.11.17v2.75H8.85V14h2.66a2.44,2.44,0,0,1,1.85.72,2.59,2.59,0,0,1,.69,1.87,2.42,2.42,0,0,1-.68,1.82,2.59,2.59,0,0,1-1.89.67Zm0-1.32h1.09a.86.86,0,0,0,.7-.29,1.31,1.31,0,0,0,.25-.86,1.55,1.55,0,0,0-.25-.93.81.81,0,0,0-.68-.35h-1.1Zm8.33,2.47H16.61l-.42,1.6H14.52L17,14h1.44l2.46,7.82H19.18ZM17,18.87h1.46l-.73-2.79Zm10.71.31a2.92,2.92,0,0,1-.75,2,2.66,2.66,0,0,1-2,.69,2.46,2.46,0,0,1-2-.87,3.87,3.87,0,0,1-.71-2.49V17.21a3.77,3.77,0,0,1,.73-2.48,2.5,2.5,0,0,1,2-.87,2.52,2.52,0,0,1,1.94.71,3.24,3.24,0,0,1,.75,2.05H26.09a2.09,2.09,0,0,0-.25-1.14,1,1,0,0,0-.86-.31,1,1,0,0,0-.9.44,3.08,3.08,0,0,0-.28,1.46v1.48a3.4,3.4,0,0,0,.26,1.6,1,1,0,0,0,.89.44,1,1,0,0,0,.86-.3,2,2,0,0,0,.27-1.1ZM35,21.78H33.43V18.44H31.1v3.35H29.52V14H31.1v3.16h2.33V14H35Zm6.19-3.38H38.73v2.07h2.91v1.31H37.15V14h4.48v1.32h-2.9v1.85h2.46Zm11.12,3.38H5
0.73l-2.31-5.13v5.13H46.84V14h1.58l2.32,5.13V14h1.57Zm3.8,0H54.53V14h1.58Zm6.24-3.19H59.89v3.19H58.31V14h4.33v1.32H59.89v2h2.46Zm3.73,3.19H64.49V14h1.58Z"/></g><g id="drop"><path id="drop-2" data-name="drop" class="cls-2" d="M80.09,22.44a7.57,7.57,0,0,1-7.57-7.57c0-3.6,2.82-7,4.75-9.57A28.33,28.33,0,0,0,80,1a.09.09,0,0,1,.16,0,28.33,28.33,0,0,0,2.74,4.32c1.93,2.52,4.75,6,4.75,9.57H83a.18.18,0,0,0-.18.18v2.6h-2.6a.18.18,0,0,0-.18.18Zm5.61-4.68V15.85H83.8v1.91Zm-2.81,2.81V18.66H81v1.91ZM81,23.06a7.17,7.17,0,0,0,1.91-.38v-.75H81Zm4.94-2A.23.23,0,0,0,86.1,21a.22.22,0,0,0,.06-.16V19.12H84.25V21Zm-1.51,1.9a7.25,7.25,0,0,0,.64-.37h-.64Zm3.39-5.16a7.18,7.18,0,0,0,.38-1.91H87.06v1.91Zm-.13,2.17a7.27,7.27,0,0,0,.37-.64h-.37ZM74.5,17.51a9.84,9.84,0,0,1,.73-6.4A27.21,27.21,0,0,1,76.75,8l.07-.11-.22,0a24.78,24.78,0,0,0-1.71,2.64,8.79,8.79,0,0,0-1.3,5.59,7,7,0,0,0,2.31,4.18A7.83,7.83,0,0,1,74.5,17.51Z"/></g></svg>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-header.svg
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-header.svg b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-header.svg
new file mode 100644
index 0000000..3e23cdd
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-header.svg
@@ -0,0 +1,17 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 119 56"><defs><style>.cls-1{fill:none;}.cls-2{fill:#fff;}.cls-3{fill:#456a7d;}.cls-4{fill:#3f3f3f;}</style></defs><title>hdf-header</title><g id="rectangle-for-sizing-2" data-name="rectangle-for-sizing"><rect class="cls-1" width="119" height="56"/></g><g id="logo-hwx"><path id="logo-hwx-2" data-name="logo-hwx" class="cls-2" d="M11,32.85V30.22H9.27v2.62H7.64V26.22H9.27v2.55H11V26.22h1.63v6.62Zm7.13-.69a2.43,2.43,0,0,1-1.85.74,2.43,2.43,0,0,1-1.85-.74,3.4,3.4,0,0,1-.67-2.62,3.4,3.4,0,0,1,.67-2.62,2.43,2.43,0,0,1,1.85-.74,2.43,2.43,0,0,1,1.85.74,3.4,3.4,0,0,1,.67,2.62A3.4,3.4,0,0,1,18.13,32.16Zm-1.2-4.25a.88.88,0,0,0-1.3,0c-.15.2-.24.42-.24,1.63s.09,1.42.24,1.62a.86.86,0,0,0,1.3,0c.15-.2.24-.41.24-1.62S17.07,28.1,16.93,27.91Zm6.35,4.94L22.1,30.38h-.55v2.47H19.92V26.22h2.62a2.12,2.12,0,0,1,2.26,2.13,1.91,1.91,0,0,1-1.08,1.73l1.43,2.76Zm-.8-5.16h-.92V29h.92a.67.67,0,1,0,0-1.34Zm6.38,0v5.17H27.23V27.67H25.6V26.22h4.88v1.45Zm6.55,4.48a2.
67,2.67,0,0,1-3.7,0A3.4,3.4,0,0,1,31,29.54a3.4,3.4,0,0,1,.67-2.62,2.67,2.67,0,0,1,3.7,0,3.4,3.4,0,0,1,.67,2.62A3.4,3.4,0,0,1,35.41,32.16Zm-1.2-4.25a.88.88,0,0,0-1.3,0c-.15.2-.24.42-.24,1.63s.09,1.42.24,1.62a.86.86,0,0,0,1.3,0c.15-.2.24-.41.24-1.62S34.35,28.1,34.21,27.91ZM41,32.85l-2.12-3.33v3.33H37.2V26.22h1.42l2.12,3.33V26.22h1.63v6.62Zm8.71,0H48.33l-1.07-3.27-1.07,3.27H44.87L43,26.22h1.7l.87,3.5,1.07-3.5h1.15l1.07,3.5.87-3.5h1.7Zm6.55-.69a2.67,2.67,0,0,1-3.7,0,3.4,3.4,0,0,1-.67-2.62,3.4,3.4,0,0,1,.67-2.62,2.67,2.67,0,0,1,3.7,0,3.4,3.4,0,0,1,.67,2.62A3.4,3.4,0,0,1,56.21,32.16ZM55,27.91a.88.88,0,0,0-1.3,0c-.15.2-.24.42-.24,1.63s.09,1.42.24,1.62a.86.86,0,0,0,1.3,0c.15-.2.24-.41.24-1.62S55.16,28.1,55,27.91Zm6.35,4.94-1.17-2.47h-.55v2.47H58V26.22h2.62a2.12,2.12,0,0,1,2.26,2.13,1.91,1.91,0,0,1-1.08,1.73l1.43,2.76Zm-.8-5.16h-.92V29h.92a.67.67,0,1,0,0-1.34Zm7.21,5.16-1.53-2.72-.56.64v2.08H64.08V26.22h1.61v2.34l1.92-2.34h2L67.33,28.9l2.34,3.95Zm4.46.06A3.23,3.23,0,0,1,69.72,32l1-1a2.16,2.1
6,0,0,0,1.48.47c.64,0,.95-.21.95-.6a.55.55,0,0,0-.13-.39.74.74,0,0,0-.44-.18l-.8-.11a2.24,2.24,0,0,1-1.33-.59A1.83,1.83,0,0,1,70,28.26a2.14,2.14,0,0,1,2.42-2.09,2.94,2.94,0,0,1,2.23.79l-1,1a1.64,1.64,0,0,0-1.26-.39c-.57,0-.8.33-.8.61a.44.44,0,0,0,.13.31.75.75,0,0,0,.46.2l.8.11a2.23,2.23,0,0,1,1.3.55,1.93,1.93,0,0,1,.49,1.45C74.79,32.17,73.62,32.9,72.23,32.9Zm3.64-5.16a.79.79,0,1,1,.79-.79A.79.79,0,0,1,75.87,27.75Zm0-1.43a.64.64,0,1,0,.63.64A.63.63,0,0,0,75.87,26.31ZM76,27.38,75.86,27h-.11v.34h-.16v-.85h.32a.26.26,0,0,1,.27.26A.24.24,0,0,1,76,27l.19.36Zm-.14-.72h-.14v.26h.14a.13.13,0,1,0,0-.26ZM58.51,20c.69,1.38-.17,4.39-.27,4.7H60a10.26,10.26,0,0,0,1.6-3.24A5.2,5.2,0,0,0,58.51,20Zm-9.07-1.49a26.57,26.57,0,0,1-1.29,6.19h1.7a8.86,8.86,0,0,0,1.75-3A5.55,5.55,0,0,0,49.44,18.53Zm19-1.4A7.79,7.79,0,0,0,66,10.93,8.28,8.28,0,0,0,63.3,10a6.86,6.86,0,0,0-3.52-1.24,1.22,1.22,0,0,0-.4,0s-.17.18-.22.25a8.15,8.15,0,0,0-1.48,4.38c.36,1.19,2.48,2.84,2.48,2.84a5.89,5.89,0,0,1-3-2.6,13.4,13.4,0,0,1,.
18-3.25h0a17.45,17.45,0,0,0-6,1.85,9.29,9.29,0,0,0-2.74,3.5,8.63,8.63,0,0,0-.76,2.68c-.12.81-.51,2.45-.51,2.45a.26.26,0,0,0,.19-.06,10.86,10.86,0,0,1,1.55-3c3.74,1.57,3.21,6.5,3.17,6.85H54a12.38,12.38,0,0,0,.81-4.3c7-3.15,9,4,9.08,4.3h1.71a25.87,25.87,0,0,0-1.69-5.63c-.91-2,.11-2.56.26-2.58A2.92,2.92,0,0,1,66,17c.52.32.65.89.79,1.62.29,1.51.51,3.41,2,3.46a1.55,1.55,0,0,0,1.6-1.23L70,20.65a.69.69,0,0,1-.71.46C68.53,21.11,68.36,20,68.42,17.14Zm-2.19-2.87a.41.41,0,1,1,.41-.41A.41.41,0,0,1,66.23,14.26ZM37.73,21c.55,1.11-.14,3.51-.21,3.76h1.37a8.21,8.21,0,0,0,1.28-2.59A4.16,4.16,0,0,0,37.73,21Zm-7.26-1.2a21.26,21.26,0,0,1-1,5H30.8a7.09,7.09,0,0,0,1.4-2.38A4.44,4.44,0,0,0,30.48,19.77Zm15.18-1.12a6.23,6.23,0,0,0-1.9-5,6.63,6.63,0,0,0-2.2-.75,5.49,5.49,0,0,0-2.81-1,1,1,0,0,0-.32,0s-.14.15-.17.2a6.52,6.52,0,0,0-1.19,3.5c.29.95,2,2.27,2,2.27a4.72,4.72,0,0,1-2.38-2.08,10.72,10.72,0,0,1,.15-2.6h0A14,14,0,0,0,32,14.76a7.43,7.43,0,0,0-2.19,2.8,6.9,6.9,0,0,0-.6,2.14c-.09.65-.41,2-.41,2a.21.21,0,0,
0,.16-.05,8.69,8.69,0,0,1,1.24-2.37c3,1.25,2.57,5.2,2.54,5.48H34.1a9.9,9.9,0,0,0,.65-3.44c5.62-2.52,7.19,3.18,7.26,3.44h1.37A20.7,20.7,0,0,0,42,20.22c-.73-1.57.09-2,.21-2.07a2.33,2.33,0,0,1,1.53.38c.42.26.52.71.63,1.29.23,1.21.41,2.73,1.63,2.77a1.24,1.24,0,0,0,1.28-1l-.42-.14a.55.55,0,0,1-.56.37C45.75,21.84,45.61,20.94,45.66,18.65Zm-1.75-2.3a.33.33,0,1,1,.33-.33A.33.33,0,0,1,43.91,16.36ZM21.1,21.69c.44.89-.11,2.81-.17,3H22a6.57,6.57,0,0,0,1-2.08A3.33,3.33,0,0,0,21.1,21.69Zm-5.8-1a17,17,0,0,1-.83,4h1.09a5.67,5.67,0,0,0,1.12-1.9A3.55,3.55,0,0,0,15.3,20.73Zm12.15-.89a5,5,0,0,0-1.52-4,5.3,5.3,0,0,0-1.76-.6,4.39,4.39,0,0,0-2.25-.8.78.78,0,0,0-.26,0s-.11.12-.14.16a5.22,5.22,0,0,0-.95,2.8c.23.76,1.59,1.82,1.59,1.82a3.77,3.77,0,0,1-1.9-1.66,8.58,8.58,0,0,1,.12-2.08h0a11.17,11.17,0,0,0-3.84,1.19A6,6,0,0,0,14.77,19a5.52,5.52,0,0,0-.48,1.71c-.07.52-.33,1.57-.33,1.57a.17.17,0,0,0,.12,0,7,7,0,0,1,1-1.9c2.4,1,2.06,4.16,2,4.39h1.09a7.92,7.92,0,0,0,.52-2.75c4.49-2,5.75,2.55,5.81,2.75h1.09a16.56,16.
56,0,0,0-1.08-3.6c-.58-1.26.07-1.64.17-1.65a1.87,1.87,0,0,1,1.22.3c.33.2.41.57.5,1,.18,1,.33,2.18,1.31,2.21a1,1,0,0,0,1-.79l-.33-.11a.44.44,0,0,1-.45.29C27.51,22.39,27.41,21.67,27.44,19.84ZM26,18a.26.26,0,1,1,.26-.26A.26.26,0,0,1,26,18Z"/></g><g id="icon-hdf"><g id="path-icon-hdf"><path class="cls-3" d="M100.22,9.18a3.86,3.86,0,0,1,1.93.52l14,8.09a3.86,3.86,0,0,1,1.93,3.34V37.32a3.86,3.86,0,0,1-1.93,3.34l-14,8.09a3.86,3.86,0,0,1-3.86,0l-14-8.09a3.86,3.86,0,0,1-1.93-3.34V21.14a3.86,3.86,0,0,1,1.93-3.34l14-8.09A3.86,3.86,0,0,1,100.22,9.18Z"/></g><path id="text-icon-hdf" class="cls-2" d="M96.37,34h-2V29.83H91.48V34h-2V24.23h2v4H94.4v-4h2ZM98,34V24.23h2.59a3.56,3.56,0,0,1,2.73,1.09,4.27,4.27,0,0,1,1,3v1.59a4.28,4.28,0,0,1-1,3,3.64,3.64,0,0,1-2.81,1.1Zm2-8.14v6.5h.59a1.66,1.66,0,0,0,1.39-.52,3.1,3.1,0,0,0,.42-1.8v-1.7a3.48,3.48,0,0,0-.38-1.91,1.55,1.55,0,0,0-1.3-.57ZM110.78,30h-3.07v4h-2V24.23h5.42v1.65h-3.44v2.51h3.07Zm2.83-5.47h-.38v1.28h-.5V24.56h-.4v-.32h1.28Zm1.53.25-.31,1h-.24l-.31
-1v1h-.45V24.23h.55l.33,1.09.36-1.09h.52v1.61h-.45Z"/></g><g id="text-dataflow"><path id="text-dataflow-2" data-name="text-dataflow" class="cls-4" d="M7.64,49V39.52H9.9a3.26,3.26,0,0,1,2.58,1,4.26,4.26,0,0,1,.92,2.92v1.59A4.18,4.18,0,0,1,12.46,48a3.46,3.46,0,0,1-2.7,1Zm1.19-8.46V48h1a2.22,2.22,0,0,0,1.84-.72,3.4,3.4,0,0,0,.59-2.13V43.45a3.44,3.44,0,0,0-.58-2.2,2.13,2.13,0,0,0-1.74-.7Zm12.15,6h-3.2L17,49H15.82l3.05-9.48h1L23,49H21.73Zm-2.89-1h2.58l-1.29-4.31Zm12.68-5H28.27V49H27.08V40.55H24.6v-1h6.17Zm6.38,6h-3.2L33.21,49H32L35,39.52h1L39.12,49H37.9Zm-2.89-1h2.58l-1.29-4.31Zm11.87-.68h-3.2V49H41.73V39.52h4.93v1H42.92v3.25h3.2ZM50.91,48h3.72v1H49.71V39.52h1.2ZM63.24,45.1a4.72,4.72,0,0,1-.83,3,2.82,2.82,0,0,1-2.35,1,2.85,2.85,0,0,1-2.31-1,4.51,4.51,0,0,1-.88-2.88V43.45a4.74,4.74,0,0,1,.84-3A2.8,2.8,0,0,1,60,39.39a2.84,2.84,0,0,1,2.33,1,4.65,4.65,0,0,1,.86,2.95ZM62,43.44a4.07,4.07,0,0,0-.49-2.24,1.89,1.89,0,0,0-3,0A3.93,3.93,0,0,0,58,43.4v1.7a4,4,0,0,0,.5,2.21,1.89,1.89,0,0,0,3,.06,3.94
,3.94,0,0,0,.5-2.15Zm6.09,2.62.12,1.06.19-.94L70,39.57h1l1.47,6.61.19,1L72.75,46l1.16-6.48H75.1L73.22,49H72.14l-1.58-6.91-.1-.56-.1.56L68.72,49H67.64l-1.88-9.48H67Z"/></g></svg>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-icon.svg
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-icon.svg b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-icon.svg
new file mode 100644
index 0000000..817e18e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hdf-icon.svg
@@ -0,0 +1,17 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 37 41"><defs><style>.cls-1{fill:#456a7d;}.cls-2{fill:#fff;}</style></defs><title>hdf-icon</title><g id="icon-hdf_standalone_" data-name="icon-hdf (standalone)"><g id="path-icon-hdf"><path class="cls-1" d="M18.5.45A3.86,3.86,0,0,1,20.43,1l14,8.09a3.86,3.86,0,0,1,1.93,3.34V28.59a3.86,3.86,0,0,1-1.93,3.34L20.43,40a3.86,3.86,0,0,1-3.86,0l-14-8.09A3.86,3.86,0,0,1,.62,28.59V12.41A3.86,3.86,0,0,1,2.55,9.06L16.57,1A3.86,3.86,0,0,1,18.5.45Z"/></g><path id="text-icon-hdf" class="cls-2" d="M14.65,25.29h-2V21.11H9.76v4.19h-2V15.51h2v4h2.92v-4h2Zm1.59,0V15.51h2.59a3.56,3.56,0,0,1,2.73,1.09,4.27,4.27,0,0,1,1,3v1.59a4.28,4.28,0,0,1-1,3,3.64,3.64,0,0,1-2.81,1.1Zm2-8.14v6.5h.59a1.66,1.66,0,0,0,1.39-.52,3.1,3.1,0,0,0,.42-1.8v-1.7a3.48,3.48,0,0,0-.38-1.91,1.55,1.55,0,0,0-1.3-.57ZM29.06,21.3H26v4H24V15.51h5.42v1.65H26v2.51h3.07Zm2.83-5.47h-.38v1.28H31V15.83h-.4v-.32H31.9Zm1.53.25-.31,1h-.24l-.31-1v1h-.45V15.51h.55L33,16.6l.36-1.09h.52v1.61h-.45Z"/></
g></svg>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hwx-logo.svg
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hwx-logo.svg b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hwx-logo.svg
new file mode 100644
index 0000000..cad16e1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/hwx-logo.svg
@@ -0,0 +1,17 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 426.52 173.3"><defs><style>.cls-1,.cls-7{fill:#fff;}.cls-2{fill:url(#New_Gradient_Swatch_1);}.cls-3{fill:url(#New_Gradient_Swatch_1-2);}.cls-4{fill:url(#New_Gradient_Swatch_1-3);}.cls-5,.cls-6{fill:#231f20;}.cls-5{opacity:0.15;}.cls-6{opacity:0.1;}.cls-7{opacity:0.2;}.cls-8{fill:url(#New_Gradient_Swatch_1-4);}.cls-9{fill:url(#New_Gradient_Swatch_1-5);}.cls-10{fill:url(#New_Gradient_Swatch_1-6);}.cls-11{fill:url(#New_Gradient_Swatch_1-7);}.cls-12{fill:url(#New_Gradient_Swatch_1-8);}.cls-13{fill:url(#New_Gradient_Swatch_1-9);}</style><linearGradient id="New_Gradient_Swatch_1" x1="342.22" y1="30.58" x2="309.85" y2="108.2" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#3fae2a"/><stop offset="0.33" stop-color="#3ea52f"/><stop offset="0.87" stop-color="#3c8d3c"/><stop offset="1" stop-color="#3b8640"/></linearGradient><linearGradient id="New_Gradien
t_Swatch_1-2" x1="276.56" y1="42.58" x2="249.24" y2="108.95" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-3" x1="338.86" y1="16.54" x2="306.11" y2="93.96" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-4" x1="208.3" y1="44.2" x2="182.41" y2="106.29" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-5" x1="155.78" y1="53.8" x2="133.92" y2="106.89" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-6" x1="205.62" y1="32.97" x2="179.41" y2="94.91" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-7" x1="101.11" y1="54.92" x2="80.4" y2="104.6" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-8" x1="59.09" y1="62.6" x2="41.61" y2="105.08" xlink:href="#New_Gradient_Swatch_1"/><linearGradient id="New_Gradient_Swatch_1-9" x1="98.97" y1="45.94" x2="78" y2="95.49" xlink:href="#New_Gradient_Swatch_1"/></defs><title
>hwx-logo</title><path class="cls-1" d="M5.59,167.38H2.24v5.8H0V158.44H5.59a4.48,4.48,0,1,1,0,8.94Zm-.12-6.94H2.24v4.91H5.47a2.46,2.46,0,1,0,0-4.91Z"/><path class="cls-1" d="M25.61,171.73a5.32,5.32,0,0,1-3.89,1.57,5.27,5.27,0,0,1-3.87-1.57c-1.45-1.45-1.43-3.08-1.43-5.92s0-4.47,1.43-5.92a5.27,5.27,0,0,1,3.87-1.57,5.32,5.32,0,0,1,3.89,1.57C27.06,161.34,27,163,27,165.81S27.06,170.28,25.61,171.73Zm-1.7-10.5a3.1,3.1,0,0,0-4.39,0c-.72.79-.87,1.61-.87,4.57s.14,3.79.87,4.57a3.1,3.1,0,0,0,4.39,0c.72-.79.87-1.61.87-4.57S24.64,162,23.91,161.24Z"/><path class="cls-1" d="M46.09,173.18H44.14l-3-10.33-3,10.33H36.21l-3.85-14.74h2.36L37.29,169l3-10.54H42L45,169l2.57-10.54h2.36Z"/><path class="cls-1" d="M56,173.18V158.44h9.42v2H58.26v4.31h6.13v2H58.26v4.43h7.18v2Z"/><path class="cls-1" d="M80.58,173.18,77.48,167H74.68v6.21H72.44V158.44h5.71a4.22,4.22,0,0,1,4.57,4.31,3.87,3.87,0,0,1-2.94,3.91l3.4,6.52ZM78,160.45H74.68v4.62H78a2.32,2.32,0,1,0,0-4.62Z"/><path class="cls-1" d="M89.94,173.18V158.44h2.24v1
4.74Z"/><path class="cls-1" d="M109,173.18l-6.81-10.37v10.37H100V158.44h2l6.81,10.35V158.44h2.24v14.74Z"/><path class="cls-1" d="M127.66,171.63a5.65,5.65,0,0,1-7.93.1c-1.45-1.45-1.43-3.08-1.43-5.92s0-4.47,1.43-5.92a5.27,5.27,0,0,1,3.87-1.57,5.18,5.18,0,0,1,5.36,4.58h-2.26a3,3,0,0,0-3.11-2.57,3,3,0,0,0-2.17.91c-.72.79-.89,1.61-.89,4.57s.17,3.81.89,4.6a3,3,0,0,0,2.17.89,3.12,3.12,0,0,0,2.4-1,3.33,3.33,0,0,0,.75-2.32v-.83H123.6v-1.9H129v2.26A5.52,5.52,0,0,1,127.66,171.63Z"/><path class="cls-1" d="M150.11,160.45v12.73h-2.24V160.45h-4.06v-2h10.35v2Z"/><path class="cls-1" d="M168.86,173.18v-6.44h-6v6.44h-2.24V158.44h2.24v6.29h6v-6.29h2.24v14.74Z"/><path class="cls-1" d="M178.89,173.18V158.44h9.42v2h-7.18v4.31h6.13v2h-6.13v4.43h7.18v2Z"/><path class="cls-1" d="M206.44,160.45v4.49h6.13v2h-6.13v6.23h-2.24V158.44h9.42v2Z"/><path class="cls-1" d="M225.3,173.3a5,5,0,0,1-5.28-5.09v-9.77h2.24v9.67a3.06,3.06,0,1,0,6.11,0v-9.67h2.24v9.77A5,5,0,0,1,225.3,173.3Z"/><path class="cls-1" d="M243.15,160.4
5v12.73h-2.24V160.45h-4.06v-2h10.35v2Z"/><path class="cls-1" d="M258.74,173.3a5,5,0,0,1-5.28-5.09v-9.77h2.24v9.67a3.06,3.06,0,1,0,6.11,0v-9.67H264v9.77A5,5,0,0,1,258.74,173.3Z"/><path class="cls-1" d="M279.73,173.18,276.63,167h-2.79v6.21H271.6V158.44h5.71a4.22,4.22,0,0,1,4.58,4.31,3.87,3.87,0,0,1-2.94,3.91l3.4,6.52Zm-2.59-12.73h-3.31v4.62h3.31a2.32,2.32,0,1,0,0-4.62Z"/><path class="cls-1" d="M289.09,173.18V158.44h9.42v2h-7.18v4.31h6.13v2h-6.13v4.43h7.18v2Z"/><path class="cls-1" d="M323.07,171.73a5.58,5.58,0,0,1-7.76,0c-1.45-1.45-1.43-3.08-1.43-5.92s0-4.47,1.43-5.92a5.58,5.58,0,0,1,7.76,0c1.45,1.45,1.41,3.08,1.41,5.92S324.52,170.28,323.07,171.73Zm-1.7-10.5a3.1,3.1,0,0,0-4.39,0c-.72.79-.87,1.61-.87,4.57s.14,3.79.87,4.57a3.1,3.1,0,0,0,4.39,0c.72-.79.87-1.61.87-4.57S322.1,162,321.37,161.24Z"/><path class="cls-1" d="M334,160.45v4.49h6.13v2H334v6.23h-2.24V158.44h9.42v2Z"/><path class="cls-1" d="M365.66,171.79a5.26,5.26,0,0,1-3.81,1.39h-5.18V158.44h5.18a5.26,5.26,0,0,1,3.81,1.39c1.59,1.59,
1.49,3.75,1.49,5.88S367.25,170.2,365.66,171.79Zm-1.53-10.35a3.19,3.19,0,0,0-2.5-1h-2.71v10.72h2.71a3.19,3.19,0,0,0,2.5-1c.77-.85.79-2.44.79-4.47S364.89,162.29,364.13,161.44Z"/><path class="cls-1" d="M382.65,173.18l-1-3h-5.82l-1,3h-2.38l5.4-14.74h1.8l5.4,14.74Zm-3.87-11.41-2.3,6.54H381Z"/><path class="cls-1" d="M394.81,160.45v12.73h-2.24V160.45h-4.06v-2h10.35v2Z"/><path class="cls-1" d="M412.57,173.18l-1-3h-5.82l-1,3h-2.38l5.4-14.74h1.8l5.4,14.74Zm-3.87-11.41-2.3,6.54h4.53Z"/><path class="cls-1" d="M418.73,159.15v3.7h-.78v-3.7h-1.18v-.71h3.15v.71Zm5.31,3.7V160l-.93,1.85h-.73l-.93-1.85v2.88h-.8v-4.41h.8l1.29,2.51,1.3-2.51h.79v4.41Z"/><path class="cls-1" d="M20.92,148.86V132.65H10.22v16.21H.17V107.93H10.22v15.75H20.92V107.93H31v40.92Z"/><path class="cls-1" d="M64.95,144.6a15,15,0,0,1-11.44,4.6,15,15,0,0,1-11.44-4.6c-4.31-4.31-4.14-10.06-4.14-16.21s-.17-11.9,4.14-16.21a15,15,0,0,1,11.44-4.6,15,15,0,0,1,11.44,4.6c4.31,4.31,4.14,10.06,4.14,16.21S69.26,140.29,64.95,144.6Zm-7.41-26.27a5.43,
5.43,0,0,0-8,0c-.92,1.21-1.5,2.59-1.5,10.06s.58,8.79,1.5,10a5.32,5.32,0,0,0,8,0c.92-1.21,1.49-2.53,1.49-10S58.45,119.54,57.53,118.34Z"/><path class="cls-1" d="M96.74,148.86,89.5,133.62H86.1v15.23H76V107.93H92.25c9.08,0,14,6.44,14,13.16a11.77,11.77,0,0,1-6.67,10.69l8.85,17.07ZM91.79,117H86.1v8.28h5.69a4.14,4.14,0,1,0,0-8.28Z"/><path class="cls-1" d="M131.23,116.9v32H121.17v-32H111.11v-9h30.18v9Z"/><path class="cls-1" d="M171.69,144.6a16.52,16.52,0,0,1-22.88,0c-4.31-4.31-4.14-10.06-4.14-16.21s-.17-11.9,4.14-16.21a16.52,16.52,0,0,1,22.88,0c4.31,4.31,4.14,10.06,4.14,16.21S176,140.29,171.69,144.6Zm-7.41-26.27a5.43,5.43,0,0,0-8,0c-.92,1.21-1.49,2.59-1.49,10.06s.57,8.79,1.49,10a5.32,5.32,0,0,0,8,0c.92-1.21,1.5-2.53,1.5-10S165.2,119.54,164.28,118.34Z"/><path class="cls-1" d="M206,148.86l-13.1-20.58v20.58H182.79V107.93h8.79l13.1,20.58V107.93h10.06v40.92Z"/><path class="cls-1" d="M259.76,148.86h-8.22l-6.61-20.18-6.61,20.18h-8.16l-11.26-40.92h10.52l5.35,21.61,6.61-21.61h7.13l6.61,21.61,5.35-21
.61H271Z"/><path class="cls-1" d="M300.23,144.6a16.52,16.52,0,0,1-22.88,0c-4.31-4.31-4.14-10.06-4.14-16.21s-.17-11.9,4.14-16.21a16.52,16.52,0,0,1,22.88,0c4.31,4.31,4.14,10.06,4.14,16.21S304.54,140.29,300.23,144.6Zm-7.41-26.27a5.43,5.43,0,0,0-8,0c-.92,1.21-1.49,2.59-1.49,10.06s.57,8.79,1.49,10a5.32,5.32,0,0,0,8,0c.92-1.21,1.5-2.53,1.5-10S293.73,119.54,292.81,118.34Z"/><path class="cls-1" d="M332,148.86l-7.24-15.23h-3.39v15.23H311.33V107.93h16.21c9.08,0,14,6.44,14,13.16a11.77,11.77,0,0,1-6.67,10.69l8.85,17.07ZM327.08,117h-5.69v8.28h5.69a4.14,4.14,0,1,0,0-8.28Z"/><path class="cls-1" d="M371.63,148.86l-9.43-16.78-3.45,4v12.82h-9.94V107.93h9.94v14.49l11.84-14.49h12.3l-14,16.55,14.43,24.37Z"/><path class="cls-1" d="M399.16,149.2c-6.26,0-11.38-1.15-15.52-5.4l6.44-6.44c2.13,2.13,5.86,2.87,9.14,2.87,4,0,5.86-1.32,5.86-3.68a3.42,3.42,0,0,0-.8-2.41,4.56,4.56,0,0,0-2.7-1.09l-4.94-.69c-3.62-.52-6.38-1.72-8.22-3.62s-2.82-4.71-2.82-8.22c0-7.47,5.63-12.93,14.94-12.93,5.86,0,10.29,1.38,13.79,4.89L40
8,118.8c-2.59-2.59-6-2.42-7.76-2.42-3.51,0-4.94,2-4.94,3.79a2.69,2.69,0,0,0,.8,1.9,4.61,4.61,0,0,0,2.82,1.21l4.94.69c3.68.52,6.32,1.67,8,3.39,2.18,2.13,3,5.17,3,9C415,144.66,407.78,149.2,399.16,149.2Z"/><path class="cls-1" d="M421.64,117.34a4.88,4.88,0,1,1,4.88-4.88A4.87,4.87,0,0,1,421.64,117.34Zm0-8.85a4,4,0,1,0,3.89,4A3.91,3.91,0,0,0,421.64,108.49Zm1,6.6L421.59,113h-.67v2.09h-1v-5.27h2a1.63,1.63,0,0,1,1.7,1.62,1.47,1.47,0,0,1-1,1.43l1.17,2.22Zm-.86-4.44h-.89v1.59h.89a.8.8,0,1,0,0-1.59Z"/><path class="cls-2" d="M314.43,69.67c4.25,8.55-1.06,27.09-1.64,29h10.57c8.78-13.5,9.91-20,9.91-20C324.19,70.09,314.43,69.67,314.43,69.67Z"/><path class="cls-3" d="M258.4,60.44c-.68,16.12-7.41,36.82-8,38.25h10.5c7.34-7.66,10.83-18.36,10.83-18.36C270.29,68.86,258.4,60.44,258.4,60.44Z"/><path class="cls-4" d="M375.63,51.81C376.17,25.5,361,13.49,361,13.49c-11.92-5.64-17-5.8-17-5.8C332.14.67,325.63,0,322.26,0a7.56,7.56,0,0,0-2.48.3s-1.06,1.14-1.33,1.55c-2.33,3.08-7.54,10.66-9.16,27,2.23,7.33,15.35,17.5
3,15.35,17.53-11.25-3.16-18.35-16-18.35-16-1.19-10.19,1.45-19.46,1.14-20.07,0,0-.1,0-.12,0-22.12,2.79-37.07,11.44-37.07,11.44-10.08,6.32-16.94,21.61-16.94,21.61s-3.14,5.77-4.67,16.55c-.71,5-3.17,15.14-3.17,15.14a1.61,1.61,0,0,0,1.2-.39c5-12.89,9.54-18.31,9.57-18.3,23.12,9.68,19.85,40.14,19.58,42.33h10.55c5-11.21,5-26.56,5-26.56,43.37-19.46,55.52,24.57,56.09,26.56H358c-.1-2.34-4.46-21.85-10.46-34.76-5.63-12.13.68-15.78,1.62-16,4.38-.78,9,1.19,11.79,2.9,3.22,2,4,5.49,4.85,10,1.78,9.32,3.14,21.09,12.6,21.37,7.21.21,9.89-6.12,9.89-7.61l-3.22-1.09a4.28,4.28,0,0,1-4.36,2.84C376.32,76.37,375.28,69.41,375.63,51.81ZM362.09,34a2.54,2.54,0,1,1,2.54-2.54A2.54,2.54,0,0,1,362.09,34Z"/><path class="cls-5" d="M258.4,60.44c-.06,1.35-.27,3.71-.39,5,8.91,9.08,10.17,23,10.17,23a55.88,55.88,0,0,0,3.57-8.08C270.29,68.86,258.4,60.44,258.4,60.44Z"/><path class="cls-6" d="M318.86,57.74c-19.81-11.44-42.94-10.52-58.7.6,18.8,11.3,15.9,38.28,15.65,40.34h10.55c5-11.21,5-26.56,5-26.56,43.36-19.46,55.52,24.55,56.0
9,26.56C345.37,82.74,335.18,67.17,318.86,57.74Z"/><path class="cls-7" d="M312.29,28.1c2-14.81,5.55-21.72,8.31-24.16a9.45,9.45,0,0,1,5.08-1.69c5.38-.13,11.32,2.18,18.24,5.43h0C332.13.67,325.63,0,322.26,0a7.56,7.56,0,0,0-2.48.3s-1.06,1.14-1.33,1.55c-2.33,3.08-7.54,10.66-9.16,27,2.23,7.33,15.35,17.53,15.35,17.53S312.16,34.1,312.29,28.1Z"/><path class="cls-6" d="M280.05,50.43C287.67,54,292,62.71,291.38,72.11c0,0,2.5-1.14,4.89-2-.8-8.91-7.67-16.71-16.22-19.7Z"/><path class="cls-5" d="M348.54,66.1c-.32-.74-.64-1.47-1-2.18-5.15-11.09-.32-15.09,1.27-15.82-1,.26-11.25,2.83-19.81.05C335.72,51.82,340.14,55.26,348.54,66.1Z"/><path class="cls-8" d="M186.07,75.47c3.4,6.84-.84,21.67-1.31,23.21h8.46c7-10.8,7.93-16,7.93-16A25.7,25.7,0,0,0,186.07,75.47Z"/><path class="cls-9" d="M141.25,68.08c-.55,12.89-5.93,29.46-6.38,30.6h8.4c5.87-6.13,8.66-14.69,8.66-14.69C150.76,74.83,141.25,68.08,141.25,68.08Z"/><path class="cls-10" d="M235,61.18c.43-21-11.73-30.65-11.73-30.65-9.53-4.51-13.58-4.64-13.58-4.64-9.48
-5.61-14.69-6.15-17.38-6.15a6,6,0,0,0-2,.24s-.85.91-1.07,1.24c-1.86,2.47-6,8.53-7.33,21.64,1.79,5.86,12.28,14,12.28,14-9-2.53-14.68-12.83-14.68-12.83-1-8.15,1.16-15.57.91-16.05,0,0-.08,0-.1,0a86.22,86.22,0,0,0-29.65,9.15c-8.06,5.06-13.55,17.29-13.55,17.29s-2.51,4.61-3.74,13.24c-.57,4-2.54,12.11-2.54,12.11a1.29,1.29,0,0,0,1-.31c4-10.31,7.64-14.65,7.65-14.64,18.5,7.74,15.88,32.11,15.66,33.87h8.44c4-9,4-21.25,4-21.25,34.7-15.57,44.42,19.66,44.87,21.25H221c-.08-1.87-3.57-17.48-8.36-27.81-4.51-9.71.54-12.63,1.3-12.76,3.5-.62,7.2.95,9.43,2.32,2.58,1.58,3.2,4.39,3.88,8,1.42,7.45,2.51,16.87,10.08,17.1a7.65,7.65,0,0,0,7.91-6.09l-2.58-.87a3.42,3.42,0,0,1-3.49,2.27C235.58,80.83,234.75,75.27,235,61.18ZM224.2,47a2,2,0,1,1,2-2A2,2,0,0,1,224.2,47Z"/><path class="cls-5" d="M141.25,68.08c0,1.08-.22,3-.31,4,7.13,7.26,8.14,18.37,8.14,18.37A44.7,44.7,0,0,0,151.93,84C150.76,74.83,141.25,68.08,141.25,68.08Z"/><path class="cls-6" d="M189.61,65.93c-15.85-9.15-34.35-8.42-47,.48,15,9,12.72,30.63,12.52,32.27h
8.44c4-9,4-21.25,4-21.25C202.32,61.86,212,97.07,212.5,98.68,210.83,85.93,202.67,73.47,189.61,65.93Z"/><path class="cls-7" d="M184.36,42.22C186,30.37,188.8,24.84,191,22.89a7.56,7.56,0,0,1,4.06-1.35c4.31-.11,9.06,1.75,14.59,4.34h0c-9.46-5.6-14.66-6.13-17.35-6.13a6,6,0,0,0-2,.24s-.85.91-1.07,1.24c-1.86,2.47-6,8.53-7.33,21.64,1.79,5.86,12.28,14,12.28,14S184.26,47,184.36,42.22Z"/><path class="cls-6" d="M158.57,60.08c6.1,2.89,9.54,9.82,9.06,17.34,0,0,2-.91,3.92-1.58-.64-7.13-6.14-13.37-13-15.76Z"/><path class="cls-5" d="M213.36,72.62c-.25-.59-.51-1.18-.78-1.74-4.12-8.87-.26-12.07,1-12.66-.79.21-9,2.27-15.84,0C203.1,61.19,206.64,63.94,213.36,72.62Z"/><path class="cls-11" d="M83.33,79.94c2.72,5.47-.68,17.34-1.05,18.57H89c5.62-8.64,6.34-12.82,6.34-12.82A20.56,20.56,0,0,0,83.33,79.94Z"/><path class="cls-12" d="M47.47,74C47,84.34,42.72,97.6,42.37,98.51h6.72A35,35,0,0,0,56,86.76C55.08,79.42,47.47,74,47.47,74Z"/><path class="cls-13" d="M122.5,68.51c.34-16.84-9.39-24.52-9.39-24.52-7.63-3.61-10.87
-3.71-10.87-3.71-7.59-4.49-11.75-4.92-13.91-4.92a4.84,4.84,0,0,0-1.59.19s-.68.73-.85,1c-1.49,2-4.83,6.82-5.86,17.31,1.43,4.69,9.82,11.22,9.82,11.22-7.2-2-11.74-10.26-11.74-10.26-.76-6.52.93-12.45.73-12.84,0,0-.06,0-.08,0a69,69,0,0,0-23.72,7.32c-6.45,4-10.84,13.83-10.84,13.83a34.11,34.11,0,0,0-3,10.59c-.45,3.2-2,9.69-2,9.69a1,1,0,0,0,.77-.25c3.17-8.25,6.11-11.72,6.12-11.71,14.8,6.2,12.7,25.69,12.53,27.09h6.75c3.22-7.18,3.22-17,3.22-17,27.76-12.46,35.54,15.72,35.9,17h6.76a102.27,102.27,0,0,0-6.69-22.25c-3.6-7.77.43-10.1,1-10.21,2.8-.5,5.76.76,7.54,1.86,2.06,1.27,2.56,3.51,3.11,6.39,1.14,6,2,13.5,8.06,13.68a6.12,6.12,0,0,0,6.33-4.87l-2.06-.7a2.74,2.74,0,0,1-2.79,1.82C122.94,84.23,122.27,79.78,122.5,68.51Zm-8.67-11.37a1.63,1.63,0,1,1,1.63-1.63A1.63,1.63,0,0,1,113.83,57.14Z"/><path class="cls-5" d="M47.47,74c0,.86-.17,2.38-.25,3.21,5.7,5.81,6.51,14.69,6.51,14.69A35.76,35.76,0,0,0,56,86.76C55.08,79.42,47.47,74,47.47,74Z"/><path class="cls-6" d="M86.16,72.3C73.49,65,58.68,65.57,48.6,72.69c
12,7.23,10.17,24.5,10,25.82h6.75c3.22-7.18,3.22-17,3.22-17,27.75-12.45,35.53,15.71,35.9,17C103.13,88.3,96.61,78.34,86.16,72.3Z"/><path class="cls-7" d="M82,53.33c1.3-9.48,3.55-13.9,5.32-15.46a6,6,0,0,1,3.25-1.08c3.45-.09,7.24,1.4,11.67,3.48h0c-7.57-4.48-11.73-4.91-13.88-4.91a4.84,4.84,0,0,0-1.59.19s-.68.73-.85,1c-1.49,2-4.83,6.82-5.86,17.31,1.43,4.69,9.82,11.22,9.82,11.22S81.88,57.18,82,53.33Z"/><path class="cls-6" d="M61.33,67.63C66.2,69.94,69,75.49,68.58,81.5c0,0,1.6-.73,3.13-1.26-.51-5.7-4.91-10.69-10.38-12.61Z"/><path class="cls-5" d="M105.16,77.65c-.2-.48-.41-.94-.62-1.39-3.29-7.1-.21-9.66.81-10.13-.63.17-7.2,1.81-12.68,0C97,68.52,99.78,70.72,105.16,77.65Z"/></svg>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/nifi-drop.svg
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/nifi-drop.svg b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/nifi-drop.svg
index c6c3def..1eee56c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/nifi-drop.svg
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/nifi-drop.svg
@@ -14,4 +14,4 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<svg id="nifi-drop" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 27 37"><defs><style>.cls-1{fill:#728e9b;}</style></defs><title>Untitled-1</title><path id="drop" class="cls-1" d="M13 35.88A12.55 12.55 0 0 1 .41 23.33c0-6 4.68-11.69 7.88-15.87A45.4 45.4 0 0 0 13 0a45.38 45.38 0 0 0 4.68 7.43c3.2 4.18 7.88 9.9 7.88 15.87h-8v4.6H13v8zm4.91-3v-4.1h-4.1v4.1h4.1zm-4.1 4.1a12.24 12.24 0 0 0 4.1-.79v-2.11h-4.1v2.85zm8.7-8.7v-4.1h-4.1v4.1h4.1zm.24 5.34l.51-.51v-3.63h-4.1v4.1h3.59zM19.39 37a12.26 12.26 0 0 0 3-1.86h-3V37zm6.38-8.73a12.21 12.21 0 0 0 .79-4.1h-2.85v4.1h2.06zm-1 4.55a12.24 12.24 0 0 0 1.86-3h-1.9v3z"/></svg>
\ No newline at end of file
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="1463.753 1191 81.404 114.21"><defs><style>.a{fill:#6a8791;}.b{fill:#ebf0f2;}</style></defs><g transform="translate(1275 1191)"><g transform="translate(188.753 0)"><path class="a" d="M235.332,110.859A39.029,39.029,0,0,1,196.3,71.83c0-18.555,14.538-36.338,24.483-49.32A146.1,146.1,0,0,0,234.927.253a.451.451,0,0,1,.811,0A146.04,146.04,0,0,0,249.88,22.51c9.941,12.979,24.484,30.765,24.484,49.32H250.547a.917.917,0,0,0-.916.916V86.131H236.25a.917.917,0,0,0-.916.916v23.812Z" transform="translate(-196.303 0)"/><path class="a" d="M252.248,104.929V92.185H240.417a.917.917,0,0,0-.916.916v11.828Z" transform="translate(-197.965 -3.545)"/><path class="a" d="M239.5,118.325a38.062,38.062,0,0,0,12.747-2.451v-6.419H240.417a.917.917,0,0,0-.916.916Z" transform="translate(-197.965 -4.21)"/><path class="a" d="M267.115,90.054V77.311H255.287a.917.917,0,0,0-.916.916V90.053h12.744Z" transform="translate(-198.537 -2.973)"/><path class="a" d="M268.737,106.559l.015-
.015a2.675,2.675,0,0,0,.766-1.89V94.581H256.773v12.747h10.07A2.671,2.671,0,0,0,268.737,106.559Z" transform="translate(-198.629 -3.638)"/><path class="a" d="M257.673,118.546a38.1,38.1,0,0,0,9.423-5.792h-9.423Z" transform="translate(-198.663 -4.337)"/><path class="a" d="M278.058,90.054a37.952,37.952,0,0,0,2.453-12.743h-8.867V90.054Z" transform="translate(-199.201 -2.973)"/><path class="a" d="M274.942,104.906a38.074,38.074,0,0,0,5.789-9.425h-5.789Z" transform="translate(-199.328 -3.672)"/><path class="a" d="M274.942,104.906a38.074,38.074,0,0,0,5.789-9.425h-5.789Z" transform="translate(-199.328 -3.672)"/></g><path class="b" d="M227.593,19.637c-.72,1.006-1.429,2.019-2.154,3.02-2.286,3.155-4.694,6.217-7.051,9.319q1.385.192,2.779.425c1.967-3.291,3.968-6.562,5.852-9.9.528-.936,1.044-1.879,1.567-2.819C229,18.938,228.041,19.012,227.593,19.637Z" transform="translate(-8.4 -0.736)"/><path class="b" d="M217.548,37.951c.346-.611.706-1.213,1.059-1.82-1.055-.2-2.111-.4-3.165-.564l-.01,0a130.66,130.6
6,0,0,0-9.138,14.021c-4.907,8.9-8.311,18.841-6.746,29.1a36.515,36.515,0,0,0,12.167,21.969,35.929,35.929,0,0,0,4.418,3.264.469.469,0,0,0,.574-.735,39.143,39.143,0,0,1-10.557-17.8c-2.92-10.773-.493-21.812,3.665-31.915A138.355,138.355,0,0,1,217.548,37.951Z" transform="translate(-7.661 -1.368)"/></g></svg>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-global-menu-controller.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-global-menu-controller.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-global-menu-controller.js
index 28d818e..7b04bed 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-global-menu-controller.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-global-menu-controller.js
@@ -316,20 +316,23 @@
var aboutDetails = response.about;
// set the document title and the about title
document.title = aboutDetails.title;
- $('#nf-version').text(aboutDetails.version);
+ $('#nf-version').text('Powered by Apache NiFi - Version ' + aboutDetails.version).attr('title', 'Powered by Apache NiFi - Version ' + aboutDetails.version);
var showVersionDetail = false;
if (aboutDetails.buildTag && aboutDetails.buildTag !== 'HEAD') {
+ $('#nf-version-detail-tag').attr('title', 'Tagged ' + aboutDetails.buildTag);
$('#nf-about-build-tag').text(aboutDetails.buildTag);
$('#nf-version-detail-tag').show();
showVersionDetail = true;
}
if (aboutDetails.buildRevision) {
+ $('#nf-version-detail-commit').attr('title', 'From ' + aboutDetails.buildRevision + ' on branch ' + aboutDetails.buildBranch);
$('#nf-about-build-revision').text(aboutDetails.buildRevision);
$('#nf-about-build-branch').text(aboutDetails.buildBranch);
$('#nf-version-detail-commit').show();
showVersionDetail = true
}
if (aboutDetails.buildTimestamp) {
+ $('#nf-version-detail-timestamp').attr('title', aboutDetails.buildTimestamp);
$('#nf-about-build-timestamp').text(aboutDetails.buildTimestamp);
$('#nf-version-detail-timestamp').show();
showVersionDetail = true;
@@ -366,30 +369,37 @@
* Initialize the modal.
*/
init: function () {
- var aboutModal = this;
+ var self = this;
var resizeAbout = function () {
var dialog = $(this);
- var top = $('#nf-about-pic-container').height() + $('.dialog-header').height() + 10; //10 for padding-top
+ var top = $('#nf-about-pic-container').height() + $('.dialog-header').height() + 20; //10 for padding-top
dialog.find('.dialog-content').css('top', top);
};
+ var openAbout = function () {
+ var dialog = $(this);
+ nf.Common.toggleScrollable(dialog.find('.dialog-content').get(0));
+ };
+
this.getElement().modal({
scrollableContentStyle: 'scrollable',
- headerText: 'About Apache NiFi',
+ headerText: 'About Hortonworks DataFlow (HDF)',
handler: {
- resize: resizeAbout
+ resize: resizeAbout,
+ open: openAbout
},
+ glasspane: "#fff",
buttons: [{
buttonText: 'Ok',
color: {
- base: '#728E9B',
- hover: '#004849',
+ base: '#3fae2a',
+ hover: '#3b8740',
text: '#ffffff'
},
handler: {
click: function () {
- aboutModal.hide();
+ self.hide();
}
}
}]
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3098192/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/views/nf-ng-breadcrumbs-directive-view.html
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/views/nf-ng-breadcrumbs-directive-view.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/views/nf-ng-breadcrumbs-directive-view.html
index 19018d3..d239884 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/views/nf-ng-breadcrumbs-directive-view.html
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/views/nf-ng-breadcrumbs-directive-view.html
@@ -32,4 +32,5 @@ limitations under the License.
</div>
<div id="breadcrumbs-right-border"></div>
</div>
+ <img src="images/hdf-footer.svg" style="position: absolute; right: 20px; width: 78px; bottom: 5px; top: 5px;">
</div>
\ No newline at end of file
[49/50] nifi git commit: NIFI-5134 Explicitly requesting UGI to
relogin before attempting to get a DB connection in HiveConnectionPool
Posted by al...@apache.org.
NIFI-5134 Explicitly requesting UGI to relogin before attempting to get a DB connection in HiveConnectionPool
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2667
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/aa4384b5
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/aa4384b5
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/aa4384b5
Branch: refs/heads/HDF-3.1-maint
Commit: aa4384b5e3632b60e00e1556d3528260a12710c0
Parents: da94eac
Author: Jeff Storck <jt...@gmail.com>
Authored: Mon Apr 30 10:39:12 2018 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Thu May 10 11:46:23 2018 -0400
----------------------------------------------------------------------
.../apache/nifi/dbcp/hive/HiveConnectionPool.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa4384b5/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
index 211494e..684f41a 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
@@ -208,8 +208,10 @@ public class HiveConnectionPool extends AbstractControllerService implements Hiv
* As of Apache NiFi 1.5.0, due to changes made to
* {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this class invoking
* {@link HiveConfigurator#authenticate(Configuration, String, String)}
- * to authenticate a principal with Kerberos, Hive controller services no longer
- * attempt relogins explicitly. For more information, please read the documentation for
+ * to authenticate a principal with Kerberos, Hive controller services no longer use a separate thread to
+ * relogin, and instead call {@link UserGroupInformation#checkTGTAndReloginFromKeytab()} from
+ * {@link HiveConnectionPool#getConnection()}. The relogin request is performed in a synchronized block to prevent
+ * threads from requesting concurrent relogins. For more information, please read the documentation for
* {@link SecurityUtil#loginKerberos(Configuration, String, String)}.
* <p/>
* In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started by
@@ -297,6 +299,15 @@ public class HiveConnectionPool extends AbstractControllerService implements Hiv
public Connection getConnection() throws ProcessException {
try {
if (ugi != null) {
+ synchronized(this) {
+ /*
+ * Make sure that only one thread can request that the UGI relogin at a time. This
+ * explicit relogin attempt is necessary due to the Hive client/thrift not implicitly handling
+ * the acquisition of a new TGT after the current one has expired.
+ * https://issues.apache.org/jira/browse/NIFI-5134
+ */
+ ugi.checkTGTAndReloginFromKeytab();
+ }
try {
return ugi.doAs((PrivilegedExceptionAction<Connection>) () -> dataSource.getConnection());
} catch (UndeclaredThrowableException e) {
[38/50] nifi git commit: NIFI-4920 Skipping sensitive properties when
updating component properties from versioned component. This closes #2505.
Posted by al...@apache.org.
NIFI-4920 Skipping sensitive properties when updating component properties from versioned component. This closes #2505.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a8e519b6
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a8e519b6
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a8e519b6
Branch: refs/heads/HDF-3.1-maint
Commit: a8e519b652340d68c3f23c25167e92af0d37867e
Parents: 87ec87a
Author: Bryan Bende <bb...@apache.org>
Authored: Fri Mar 2 10:52:22 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Mar 6 09:44:39 2018 -0500
----------------------------------------------------------------------
.../org/apache/nifi/groups/StandardProcessGroup.java | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8e519b6/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index ae9b43f..9fd5ad7 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -4098,9 +4098,17 @@ public final class StandardProcessGroup implements ProcessGroup {
private Map<String, String> populatePropertiesMap(final Map<PropertyDescriptor, String> currentProperties, final Map<String, String> proposedProperties,
final Map<String, VersionedPropertyDescriptor> proposedDescriptors, final ProcessGroup group) {
+ // since VersionedPropertyDescriptor currently doesn't know if it is sensitive or not,
+ // keep track of which property descriptors are sensitive from the current properties
+ final Set<String> sensitiveProperties = new HashSet<>();
+
final Map<String, String> fullPropertyMap = new HashMap<>();
for (final PropertyDescriptor property : currentProperties.keySet()) {
- fullPropertyMap.put(property.getName(), null);
+ if (property.isSensitive()) {
+ sensitiveProperties.add(property.getName());
+ } else {
+ fullPropertyMap.put(property.getName(), null);
+ }
}
if (proposedProperties != null) {
@@ -4108,6 +4116,11 @@ public final class StandardProcessGroup implements ProcessGroup {
final String propertyName = entry.getKey();
final VersionedPropertyDescriptor descriptor = proposedDescriptors.get(propertyName);
+ // skip any sensitive properties so we can retain whatever is currently set
+ if (sensitiveProperties.contains(propertyName)) {
+ continue;
+ }
+
String value;
if (descriptor != null && descriptor.getIdentifiesControllerService()) {
// Property identifies a Controller Service. So the value that we want to assign is not the value given.
[41/50] nifi git commit: NIFI-5012: When connecting to cluster,
esure that controller services appropriately enabled/disabled
Posted by al...@apache.org.
NIFI-5012: When connecting to cluster, esure that controller services appropriately enabled/disabled
This closes #2579.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ae6ad055
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ae6ad055
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ae6ad055
Branch: refs/heads/HDF-3.1-maint
Commit: ae6ad0556f78efa649485ce3da52d642976614fb
Parents: 79e4402
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Mar 23 15:44:06 2018 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Apr 3 15:51:11 2018 -0400
----------------------------------------------------------------------
.../controller/StandardFlowSynchronizer.java | 44 ++++++++++++++++++++
1 file changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/ae6ad055/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
index 425110c..8a79acf 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java
@@ -73,6 +73,7 @@ import org.apache.nifi.controller.serialization.FlowSynchronizer;
import org.apache.nifi.controller.serialization.StandardFlowSerializer;
import org.apache.nifi.controller.service.ControllerServiceLoader;
import org.apache.nifi.controller.service.ControllerServiceNode;
+import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.encrypt.StringEncryptor;
import org.apache.nifi.events.BulletinFactory;
import org.apache.nifi.fingerprint.FingerprintException;
@@ -749,6 +750,19 @@ public class StandardFlowSynchronizer implements FlowSynchronizer {
}
}
+ private ControllerServiceState getFinalTransitionState(final ControllerServiceState state) {
+ switch (state) {
+ case DISABLED:
+ case DISABLING:
+ return ControllerServiceState.DISABLED;
+ case ENABLED:
+ case ENABLING:
+ return ControllerServiceState.ENABLED;
+ default:
+ throw new AssertionError();
+ }
+ }
+
private ProcessGroup updateProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement,
final StringEncryptor encryptor, final FlowEncodingVersion encodingVersion) throws ProcessorInstantiationException {
@@ -779,6 +793,36 @@ public class StandardFlowSynchronizer implements FlowSynchronizer {
// get the real process group and ID
final ProcessGroup processGroup = controller.getGroup(processGroupDto.getId());
+ // determine the scheduled state of all of the Controller Service
+ final List<Element> controllerServiceNodeList = getChildrenByTagName(processGroupElement, "controllerService");
+ final Set<ControllerServiceNode> toDisable = new HashSet<>();
+ final Set<ControllerServiceNode> toEnable = new HashSet<>();
+
+ for (final Element serviceElement : controllerServiceNodeList) {
+ final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(serviceElement, encryptor);
+ final ControllerServiceNode serviceNode = processGroup.getControllerService(dto.getId());
+
+ // Check if the controller service is in the correct state. We consider it the correct state if
+ // we are in a transitional state and heading in the right direction or already in the correct state.
+ // E.g., it is the correct state if it should be 'DISABLED' and it is either DISABLED or DISABLING.
+ final ControllerServiceState serviceState = getFinalTransitionState(serviceNode.getState());
+ final ControllerServiceState clusterState = getFinalTransitionState(ControllerServiceState.valueOf(dto.getState()));
+
+ if (serviceState != clusterState) {
+ switch (clusterState) {
+ case DISABLED:
+ toDisable.add(serviceNode);
+ break;
+ case ENABLED:
+ toEnable.add(serviceNode);
+ break;
+ }
+ }
+ }
+
+ controller.disableControllerServicesAsync(toDisable);
+ controller.enableControllerServices(toEnable);
+
// processors & ports cannot be updated - they must be the same. Except for the scheduled state.
final List<Element> processorNodeList = getChildrenByTagName(processGroupElement, "processor");
for (final Element processorElement : processorNodeList) {
[32/50] nifi git commit: NIFI-4858: Expose Request Timeout from HTTP
Context Map and use that to set as the timeout of the 'AsyncContext' in
HandleHttpRequest. Otherwise,
the request will never timeout (which is OK because the HttpContextMap will
handle
Posted by al...@apache.org.
NIFI-4858: Expose Request Timeout from HTTP Context Map and use that to set as the timeout of the 'AsyncContext' in HandleHttpRequest. Otherwise, the request will never timeout (which is OK because the HttpContextMap will handle this). However, Jetty behind the scenes is adding a task to Scheduled Executor for each request with a delay of whatever the timeout is set to. Since it's currently set to Long.MAX_VALUE, that task will never be run and as a result the ExecutorService's queue will grow indefinitely, eventually exhausting the JVM Heap
This closes #2460.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a4d3bc00
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a4d3bc00
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a4d3bc00
Branch: refs/heads/HDF-3.1-maint
Commit: a4d3bc0090b0f0e103600791a689f41b070cb72e
Parents: d17b997
Author: Mark Payne <ma...@hotmail.com>
Authored: Thu Feb 8 15:02:06 2018 -0500
Committer: Bryan Bende <bb...@apache.org>
Committed: Thu Feb 8 17:42:28 2018 -0500
----------------------------------------------------------------------
.../apache/nifi/processors/standard/HandleHttpRequest.java | 4 +++-
.../nifi/processors/standard/TestHandleHttpRequest.java | 6 ++++++
.../nifi/processors/standard/TestHandleHttpResponse.java | 6 ++++++
.../src/main/java/org/apache/nifi/http/HttpContextMap.java | 9 +++++++++
.../java/org/apache/nifi/http/StandardHttpContextMap.java | 5 +++++
5 files changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d3bc00/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index d436312..5a09dab 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -281,6 +281,8 @@ public class HandleHttpRequest extends AbstractProcessor {
final String host = context.getProperty(HOSTNAME).getValue();
final int port = context.getProperty(PORT).asInteger();
final SSLContextService sslService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);
+ final HttpContextMap httpContextMap = context.getProperty(HTTP_CONTEXT_MAP).asControllerService(HttpContextMap.class);
+ final long requestTimeout = httpContextMap.getRequestTimeout(TimeUnit.MILLISECONDS);
final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
final boolean need;
@@ -404,7 +406,7 @@ public class HandleHttpRequest extends AbstractProcessor {
// Right now, that information, though, is only in the ProcessSession, not the ProcessContext,
// so it is not known to us. Should see if it can be added to the ProcessContext.
final AsyncContext async = baseRequest.startAsync();
- async.setTimeout(Long.MAX_VALUE); // timeout is handled by HttpContextMap
+ async.setTimeout(requestTimeout);
final boolean added = containerQueue.offer(new HttpRequestContainer(request, response, async));
if (added) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d3bc00/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
index 0315ba2..19e147e 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
@@ -304,5 +305,10 @@ public class TestHandleHttpRequest {
public void setRegisterSuccessfully(boolean registerSuccessfully) {
this.registerSuccessfully = registerSuccessfully;
}
+
+ @Override
+ public long getRequestTimeout(TimeUnit timeUnit) {
+ return timeUnit.convert(30000, TimeUnit.MILLISECONDS);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d3bc00/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
index d9fc0b5..d492485 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.servlet.AsyncContext;
@@ -268,5 +269,10 @@ public class TestHandleHttpResponse {
public int getCompletionCount() {
return completedCount.get();
}
+
+ @Override
+ public long getRequestTimeout(TimeUnit timeUnit) {
+ return timeUnit.convert(30000, TimeUnit.MILLISECONDS);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d3bc00/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java b/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
index 0dcff03..d050be9 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
@@ -16,6 +16,8 @@
*/
package org.apache.nifi.http;
+import java.util.concurrent.TimeUnit;
+
import javax.servlet.AsyncContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -66,4 +68,11 @@ public interface HttpContextMap extends ControllerService {
*/
void complete(String identifier);
+ /**
+ * Returns the configured timeout for HTTP Requests
+ *
+ * @param timeUnit the desired time unit
+ * @return the configured timeout for HTTP Requests
+ */
+ long getRequestTimeout(TimeUnit timeUnit);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d3bc00/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java b/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
index 88ce51a..ba70e0c 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
@@ -194,4 +194,9 @@ public class StandardHttpContextMap extends AbstractControllerService implements
}
}
}
+
+ @Override
+ public long getRequestTimeout(final TimeUnit timeUnit) {
+ return timeUnit.convert(maxRequestNanos, TimeUnit.NANOSECONDS);
+ }
}
[36/50] nifi git commit: NIFI-4866: Fixed HBase performance issue
Posted by al...@apache.org.
NIFI-4866: Fixed HBase performance issue
HBase_1_1_2_ClientService performance dropped due to accessing HBase
admin upon every FlowFile transfer. Fixed that by getting master address
only once when the service is enabled and a connection is established.
This closes #2464
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2d85d22c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2d85d22c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2d85d22c
Branch: refs/heads/HDF-3.1-maint
Commit: 2d85d22c752e81d5e637fe96b400324a78b82960
Parents: 33c44b5
Author: Koji Kawamura <ij...@apache.org>
Authored: Sat Feb 10 11:27:17 2018 +0900
Committer: Matt Gilman <ma...@gmail.com>
Committed: Sun Feb 11 16:17:24 2018 -0500
----------------------------------------------------------------------
.../org/apache/nifi/hbase/HBase_1_1_2_ClientService.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2d85d22c/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
index 12309d6..cc6927b 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
@@ -91,6 +91,7 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
private volatile Connection connection;
private volatile UserGroupInformation ugi;
+ private volatile String masterAddress;
private List<PropertyDescriptor> properties;
private KerberosProperties kerberosProperties;
@@ -212,6 +213,7 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
final Admin admin = this.connection.getAdmin();
if (admin != null) {
admin.listTableNames();
+ masterAddress = admin.getClusterStatus().getMaster().getHostAndPort();
}
}
}
@@ -549,11 +551,7 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
logger.warn("Connection has not been established, could not create a transit URI. Returning null.");
return null;
}
- try {
- final String masterAddress = connection.getAdmin().getClusterStatus().getMaster().getHostAndPort();
- return "hbase://" + masterAddress + "/" + tableName + (rowKey != null && !rowKey.isEmpty() ? "/" + rowKey : "");
- } catch (IOException e) {
- throw new RuntimeException("Failed to get HBase Admin interface, due to " + e, e);
- }
+ final String transitUriMasterAddress = StringUtils.isEmpty(masterAddress) ? "unknown" : masterAddress;
+ return "hbase://" + transitUriMasterAddress + "/" + tableName + (StringUtils.isEmpty(rowKey) ? "" : "/" + rowKey);
}
}
[39/50] nifi git commit: NIFI-4884 Fixing ordering during import from
registry so that we set schedule strategy on a processor before setting
scheduling duration. This closes #2472
Posted by al...@apache.org.
NIFI-4884 Fixing ordering during import from registry so that we set schedule strategy on a processor before setting scheduling duration. This closes #2472
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4adb5b7f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4adb5b7f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4adb5b7f
Branch: refs/heads/HDF-3.1-maint
Commit: 4adb5b7f6f3d12b36acf57c5be4f7bd8ffa66cc2
Parents: a8e519b
Author: Bryan Bende <bb...@apache.org>
Authored: Thu Feb 15 12:42:47 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Mar 13 13:33:14 2018 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/nifi/groups/StandardProcessGroup.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/4adb5b7f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index 9fd5ad7..4177917 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -4080,8 +4080,8 @@ public final class StandardProcessGroup implements ProcessGroup {
final Map<String, String> properties = populatePropertiesMap(processor.getProperties(), proposed.getProperties(), proposed.getPropertyDescriptors(), processor.getProcessGroup());
processor.setProperties(properties, true);
processor.setRunDuration(proposed.getRunDurationMillis(), TimeUnit.MILLISECONDS);
- processor.setScheduldingPeriod(proposed.getSchedulingPeriod());
processor.setSchedulingStrategy(SchedulingStrategy.valueOf(proposed.getSchedulingStrategy()));
+ processor.setScheduldingPeriod(proposed.getSchedulingPeriod());
processor.setStyle(proposed.getStyle());
processor.setYieldPeriod(proposed.getYieldDuration());
processor.setPosition(new Position(proposed.getPosition().getX(), proposed.getPosition().getY()));
[23/50] nifi git commit: RMP-10066: Add Druid components to assembly
Posted by al...@apache.org.
RMP-10066: Add Druid components to assembly
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b32e1a08
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b32e1a08
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b32e1a08
Branch: refs/heads/HDF-3.1-maint
Commit: b32e1a08029b3d02b718b05c209a0a6dec4ea1d9
Parents: d5f3eef
Author: Matthew Burgess <ma...@apache.org>
Authored: Fri Feb 2 10:34:26 2018 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Fri Feb 2 10:34:26 2018 -0500
----------------------------------------------------------------------
nifi-assembly/pom.xml | 10 ++++++++++
pom.xml | 12 ++++++++++++
2 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b32e1a08/nifi-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
index 692dd6d..3a58e6a 100755
--- a/nifi-assembly/pom.xml
+++ b/nifi-assembly/pom.xml
@@ -561,6 +561,16 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-livy-nar</artifactId>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service-api-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-nar</artifactId>
+ <type>nar</type>
+ </dependency>
</dependencies>
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b32e1a08/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b762c5c..3b2f1b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1506,6 +1506,18 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-controller-service-api-nar</artifactId>
+ <version>1.5.0</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-druid-nar</artifactId>
+ <version>1.5.0</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
<version>1.5.0</version>
</dependency>
[33/50] nifi git commit: NIFI-4830: Fixed logic errors in BLOB/CLOB
processing in JdbcCommon
Posted by al...@apache.org.
NIFI-4830: Fixed logic errors in BLOB/CLOB processing in JdbcCommon
This closes #2459.
Signed-off-by: Koji Kawamura <ij...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/da8bf834
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/da8bf834
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/da8bf834
Branch: refs/heads/HDF-3.1-maint
Commit: da8bf834538d893af3027a358ddef6263e792629
Parents: a4d3bc0
Author: Matthew Burgess <ma...@apache.org>
Authored: Thu Feb 8 14:02:58 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Feb 9 07:51:11 2018 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/util/JdbcCommon.java | 4 ++--
.../processors/standard/util/TestJdbcCommon.java | 16 +++++++++++++---
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/da8bf834/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
index 1a3714c..f8e88ef 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
@@ -280,7 +280,7 @@ public class JdbcCommon {
InputStream is = clob.getAsciiStream();
int index = 0;
int c = is.read();
- while (c > 0) {
+ while (c >= 0) {
buffer[index++] = (char) c;
c = is.read();
}
@@ -316,7 +316,7 @@ public class JdbcCommon {
InputStream is = blob.getBinaryStream();
int index = 0;
int c = is.read();
- while (c > 0) {
+ while (c >= 0) {
buffer[index++] = (byte) c;
c = is.read();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/da8bf834/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java
index afe7788..5eca32a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java
@@ -476,11 +476,13 @@ public class TestJdbcCommon {
public void testClob() throws Exception {
try (final Statement stmt = con.createStatement()) {
stmt.executeUpdate("CREATE TABLE clobtest (id INT, text CLOB(64 K))");
- stmt.execute("INSERT INTO blobtest VALUES (41, NULL)");
+ stmt.execute("INSERT INTO clobtest VALUES (41, NULL)");
PreparedStatement ps = con.prepareStatement("INSERT INTO clobtest VALUES (?, ?)");
ps.setInt(1, 42);
final char[] buffer = new char[4002];
IntStream.range(0, 4002).forEach((i) -> buffer[i] = String.valueOf(i % 10).charAt(0));
+ // Put a zero-byte in to test the buffer building logic
+ buffer[1] = 0;
ReaderInputStream isr = new ReaderInputStream(new CharArrayReader(buffer), Charset.defaultCharset());
// - set the value of the input parameter to the input stream
@@ -513,7 +515,10 @@ public class TestJdbcCommon {
assertNull(o);
} else {
assertNotNull(o);
- assertEquals(4002, o.toString().length());
+ final String text = o.toString();
+ assertEquals(4002, text.length());
+ // Third character should be '2'
+ assertEquals('2', text.charAt(2));
}
}
}
@@ -529,6 +534,8 @@ public class TestJdbcCommon {
ps.setInt(1, 42);
final byte[] buffer = new byte[4002];
IntStream.range(0, 4002).forEach((i) -> buffer[i] = (byte) ((i % 10) + 65));
+ // Put a zero-byte in to test the buffer building logic
+ buffer[1] = 0;
ByteArrayInputStream bais = new ByteArrayInputStream(buffer);
// - set the value of the input parameter to the input stream
@@ -562,7 +569,10 @@ public class TestJdbcCommon {
} else {
assertNotNull(o);
assertTrue(o instanceof ByteBuffer);
- assertEquals(4002, ((ByteBuffer) o).array().length);
+ final byte[] blob = ((ByteBuffer) o).array();
+ assertEquals(4002, blob.length);
+ // Third byte should be 67 ('C')
+ assertEquals('C', blob[2]);
}
}
}
[47/50] nifi git commit: NIFI-4395 GenerateTableFetch can't fetch
column type by state after instance reboot
Posted by al...@apache.org.
NIFI-4395 GenerateTableFetch can't fetch column type by state after instance reboot
Add multiple states recover
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2442
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/fd8f1bf9
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/fd8f1bf9
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/fd8f1bf9
Branch: refs/heads/HDF-3.1-maint
Commit: fd8f1bf9083d2baf4d43e0666c774a8d5f83e2d8
Parents: b0142dc
Author: Deon Huang <yj...@gmail.com>
Authored: Mon Jan 29 23:29:21 2018 +0800
Committer: Matt Gilman <ma...@gmail.com>
Committed: Thu May 10 11:11:43 2018 -0400
----------------------------------------------------------------------
.../processors/standard/GenerateTableFetch.java | 8 +--
.../standard/TestGenerateTableFetch.java | 61 ++++++++++++++++++++
2 files changed, 63 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/fd8f1bf9/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
index 8f535b3..188e282 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateTableFetch.java
@@ -245,7 +245,7 @@ public class GenerateTableFetch extends AbstractDatabaseFetchProcessor {
maxValueSelectColumns.add("MAX(" + colName + ") " + colName);
String maxValue = getColumnStateMaxValue(tableName, statePropertyMap, colName);
if (!StringUtils.isEmpty(maxValue)) {
- if(columnTypeMap.isEmpty()){
+ if(columnTypeMap.isEmpty() || getColumnType(tableName, colName) == null){
// This means column type cache is clean after instance reboot. We should re-cache column type
super.setup(context, false, finalFileToProcess);
}
@@ -327,7 +327,7 @@ public class GenerateTableFetch extends AbstractDatabaseFetchProcessor {
maxValueSelectColumns.add("MAX(" + colName + ") " + colName);
String maxValue = getColumnStateMaxValue(tableName, statePropertyMap, colName);
if (!StringUtils.isEmpty(maxValue)) {
- if(columnTypeMap.isEmpty()){
+ if(columnTypeMap.isEmpty() || getColumnType(tableName, colName) == null){
// This means column type cache is clean after instance reboot. We should re-cache column type
super.setup(context, false, finalFileToProcess);
}
@@ -419,10 +419,6 @@ public class GenerateTableFetch extends AbstractDatabaseFetchProcessor {
// If the table name is static and the fully-qualified key was not found, try just the column name
type = columnTypeMap.get(getStateKey(null, colName));
}
- if (type == null) {
- // This shouldn't happen as we are populating columnTypeMap when the processor is scheduled or when the first maximum is observed
- throw new ProcessException("No column type cache found for: " + colName);
- }
return type;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/fd8f1bf9/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
index 117b47e..253f4d0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java
@@ -1191,6 +1191,67 @@ public class TestGenerateTableFetch {
runner.clearTransferState();
}
+ @Test
+ public void testMultipleColumnTypeMissing() throws ClassNotFoundException, SQLException, InitializationException, IOException {
+
+ // Load test data to database
+ final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection();
+ Statement stmt = con.createStatement();
+
+ try {
+ stmt.execute("drop table TEST_QUERY_DB_TABLE");
+ stmt.execute("drop table TEST_QUERY_DB_TABLE_2");
+ } catch (final SQLException sqle) {
+ // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842]
+ }
+
+ // Create multiple table to invoke processor state stored
+ stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, bucket integer not null)");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, bucket) VALUES (1, 0)");
+ stmt.execute("create table TEST_QUERY_DB_TABLE_2 (id integer not null, bucket integer not null)");
+ stmt.execute("insert into TEST_QUERY_DB_TABLE_2 (id, bucket) VALUES (1, 0)");
+
+ runner.setProperty(GenerateTableFetch.TABLE_NAME, "${tableName}");
+ runner.setIncomingConnection(true);
+ runner.setProperty(GenerateTableFetch.MAX_VALUE_COLUMN_NAMES, "${maxValueCol}");
+
+ runner.enqueue("".getBytes(), new HashMap<String, String>() {{
+ put("tableName", "TEST_QUERY_DB_TABLE");
+ put("maxValueCol", "id");
+ }});
+
+ runner.enqueue("".getBytes(), new HashMap<String, String>() {{
+ put("tableName", "TEST_QUERY_DB_TABLE_2");
+ put("maxValueCol", "id");
+ }});
+ runner.run(2);
+ runner.assertAllFlowFilesTransferred(REL_SUCCESS, 2);
+
+ assertEquals(2,processor.columnTypeMap.size());
+ runner.clearTransferState();
+
+
+ // Remove one element from columnTypeMap to simulate it's re-cache partial state
+ Map.Entry<String,Integer> entry = processor.columnTypeMap.entrySet().iterator().next();
+ String key = entry.getKey();
+ processor.columnTypeMap.remove(key);
+
+ // Insert new records
+ stmt.execute("insert into TEST_QUERY_DB_TABLE (id, bucket) VALUES (2, 0)");
+
+ // Re-launch FlowFile to se if re-cache column type works
+ runner.enqueue("".getBytes(), new HashMap<String, String>() {{
+ put("tableName", "TEST_QUERY_DB_TABLE");
+ put("maxValueCol", "id");
+ }});
+
+ // It should re-cache column type
+ runner.run();
+ runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
+ assertEquals(2,processor.columnTypeMap.size());
+ runner.clearTransferState();
+ }
+
/**
* Simple implementation only for GenerateTableFetch processor testing.
*/
[34/50] nifi git commit: NIFI-4841 Fixing NPE when reverting local
changes involving remote group ports. This closes #2454.
Posted by al...@apache.org.
NIFI-4841 Fixing NPE when reverting local changes involving remote group ports. This closes #2454.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/828dd799
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/828dd799
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/828dd799
Branch: refs/heads/HDF-3.1-maint
Commit: 828dd799996c137c71edb783231c6e118853620d
Parents: da8bf83
Author: Bryan Bende <bb...@apache.org>
Authored: Tue Feb 6 17:43:59 2018 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Feb 9 10:33:10 2018 -0500
----------------------------------------------------------------------
.../nifi/groups/StandardProcessGroup.java | 25 ++++++-
.../apache/nifi/util/FlowDifferenceFilters.java | 52 ++++++++++++++
.../nifi/util/TestFlowDifferenceFilters.java | 76 ++++++++++++++++++++
.../nifi/web/StandardNiFiServiceFacade.java | 7 ++
.../apache/nifi/web/api/VersionsResource.java | 4 +-
.../org/apache/nifi/web/api/dto/DtoFactory.java | 6 ++
.../nifi/web/util/AffectedComponentUtils.java | 5 +-
7 files changed, 169 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index 15f2b5f..8b7dcd2 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -110,6 +110,7 @@ import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor;
import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol;
import org.apache.nifi.scheduling.ExecutionNode;
import org.apache.nifi.scheduling.SchedulingStrategy;
+import org.apache.nifi.util.FlowDifferenceFilters;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.ReflectionUtils;
import org.apache.nifi.util.SnippetUtils;
@@ -1649,6 +1650,18 @@ public final class StandardProcessGroup implements ProcessGroup {
return funnel;
}
+ for (final RemoteProcessGroup remoteProcessGroup : group.getRemoteProcessGroups()) {
+ final RemoteGroupPort remoteInputPort = remoteProcessGroup.getInputPort(identifier);
+ if (remoteInputPort != null) {
+ return remoteInputPort;
+ }
+
+ final RemoteGroupPort remoteOutputPort = remoteProcessGroup.getOutputPort(identifier);
+ if (remoteOutputPort != null) {
+ return remoteOutputPort;
+ }
+ }
+
for (final ProcessGroup childGroup : group.getProcessGroups()) {
final Connectable childGroupConnectable = findLocalConnectable(identifier, childGroup);
if (childGroupConnectable != null) {
@@ -3257,6 +3270,11 @@ public final class StandardProcessGroup implements ProcessGroup {
continue;
}
+ // Ignore differences for adding a remote port
+ if (FlowDifferenceFilters.isAddedRemotePort(diff)) {
+ continue;
+ }
+
// If this update adds a new Controller Service, then we need to check if the service already exists at a higher level
// and if so compare our VersionedControllerService to the existing service.
if (diff.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
@@ -3901,7 +3919,7 @@ public final class StandardProcessGroup implements ProcessGroup {
final String rpgId = connectableComponent.getGroupId();
final Optional<RemoteProcessGroup> rpgOption = group.getRemoteProcessGroups().stream()
.filter(component -> component.getVersionedComponentId().isPresent())
- .filter(component -> id.equals(component.getVersionedComponentId().get()))
+ .filter(component -> rpgId.equals(component.getVersionedComponentId().get()))
.findAny();
if (!rpgOption.isPresent()) {
@@ -4197,8 +4215,9 @@ public final class StandardProcessGroup implements ProcessGroup {
final FlowComparator flowComparator = new StandardFlowComparator(snapshotFlow, currentFlow, getAncestorGroupServiceIds(), new EvolvingDifferenceDescriptor());
final FlowComparison comparison = flowComparator.compare();
final Set<FlowDifference> differences = comparison.getDifferences().stream()
- .filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
- .collect(Collectors.toCollection(HashSet::new));
+ .filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
+ .filter(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS)
+ .collect(Collectors.toCollection(HashSet::new));
LOG.debug("There are {} differences between this Local Flow and the Versioned Flow: {}", differences.size(), differences);
return differences;
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
new file mode 100644
index 0000000..ca48b99
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.registry.flow.ComponentType;
+import org.apache.nifi.registry.flow.VersionedComponent;
+import org.apache.nifi.registry.flow.diff.DifferenceType;
+import org.apache.nifi.registry.flow.diff.FlowDifference;
+import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedComponent;
+
+import java.util.function.Predicate;
+
+public class FlowDifferenceFilters {
+
+ /**
+ * Predicate that returns true if the difference is NOT a remote port being added, and false if it is.
+ */
+ public static Predicate<FlowDifference> FILTER_ADDED_REMOTE_PORTS = (fd) -> {
+ return !isAddedRemotePort(fd);
+ };
+
+ public static boolean isAddedRemotePort(final FlowDifference fd) {
+ if (fd.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
+ VersionedComponent component = fd.getComponentA();
+ if (component == null || fd.getComponentB() instanceof InstantiatedVersionedComponent) {
+ component = fd.getComponentB();
+ }
+
+ if (component.getComponentType() == ComponentType.REMOTE_INPUT_PORT
+ || component.getComponentType() == ComponentType.REMOTE_OUTPUT_PORT) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
new file mode 100644
index 0000000..ee65816
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.registry.flow.ComponentType;
+import org.apache.nifi.registry.flow.VersionedProcessor;
+import org.apache.nifi.registry.flow.VersionedRemoteGroupPort;
+import org.apache.nifi.registry.flow.diff.DifferenceType;
+import org.apache.nifi.registry.flow.diff.StandardFlowDifference;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestFlowDifferenceFilters {
+
+ @Test
+ public void testFilterAddedRemotePortsWithRemoteInputPortAsComponentB() {
+ VersionedRemoteGroupPort remoteGroupPort = new VersionedRemoteGroupPort();
+ remoteGroupPort.setComponentType(ComponentType.REMOTE_INPUT_PORT);
+
+ StandardFlowDifference flowDifference = new StandardFlowDifference(
+ DifferenceType.COMPONENT_ADDED, null, remoteGroupPort, null, null, "");
+
+ // predicate should return false because we don't want to include changes for adding a remote input port
+ Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ }
+
+ @Test
+ public void testFilterAddedRemotePortsWithRemoteInputPortAsComponentA() {
+ VersionedRemoteGroupPort remoteGroupPort = new VersionedRemoteGroupPort();
+ remoteGroupPort.setComponentType(ComponentType.REMOTE_INPUT_PORT);
+
+ StandardFlowDifference flowDifference = new StandardFlowDifference(
+ DifferenceType.COMPONENT_ADDED, remoteGroupPort, null, null, null, "");
+
+ // predicate should return false because we don't want to include changes for adding a remote input port
+ Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ }
+
+ @Test
+ public void testFilterAddedRemotePortsWithRemoteOutputPort() {
+ VersionedRemoteGroupPort remoteGroupPort = new VersionedRemoteGroupPort();
+ remoteGroupPort.setComponentType(ComponentType.REMOTE_OUTPUT_PORT);
+
+ StandardFlowDifference flowDifference = new StandardFlowDifference(
+ DifferenceType.COMPONENT_ADDED, null, remoteGroupPort, null, null, "");
+
+ // predicate should return false because we don't want to include changes for adding a remote input port
+ Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ }
+
+ @Test
+ public void testFilterAddedRemotePortsWithNonRemoteInputPort() {
+ VersionedProcessor versionedProcessor = new VersionedProcessor();
+ versionedProcessor.setComponentType(ComponentType.PROCESSOR);
+
+ StandardFlowDifference flowDifference = new StandardFlowDifference(
+ DifferenceType.COMPONENT_ADDED, null, versionedProcessor, null, null, "");
+
+ // predicate should return true because we do want to include changes for adding a non-port
+ Assert.assertTrue(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index 65f6329..bf794cf 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -122,6 +122,7 @@ import org.apache.nifi.reporting.Bulletin;
import org.apache.nifi.reporting.BulletinQuery;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.ComponentType;
+import org.apache.nifi.util.FlowDifferenceFilters;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.api.dto.AccessPolicyDTO;
import org.apache.nifi.web.api.dto.AccessPolicySummaryDTO;
@@ -3965,6 +3966,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream()
.filter(difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED) // components that are added are not components that will be affected in the local flow.
.filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
+ .filter(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS)
.map(difference -> {
final VersionedComponent localComponent = difference.getComponentA();
@@ -4001,6 +4003,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
continue;
}
+ // Ignore differences for adding remote ports
+ if (FlowDifferenceFilters.isAddedRemotePort(difference)) {
+ continue;
+ }
+
final VersionedComponent localComponent = difference.getComponentA();
if (localComponent == null) {
continue;
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
index d8c8ddf..8028bd9 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
@@ -1539,7 +1539,9 @@ public class VersionsResource extends ApplicationResource {
for (final AffectedComponentEntity original : originalEntities) {
try {
final AffectedComponentEntity updatedEntity = AffectedComponentUtils.updateEntity(original, serviceFacade, dtoFactory, user);
- entities.add(updatedEntity);
+ if (updatedEntity != null) {
+ entities.add(updatedEntity);
+ }
} catch (final ResourceNotFoundException rnfe) {
// Component was removed. Just continue on without adding anything to the entities.
// We do this because the intent is to get updated versions of the entities with current
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 36da701..2ac31fb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -139,6 +139,7 @@ import org.apache.nifi.reporting.Bulletin;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.ReportingTask;
import org.apache.nifi.scheduling.SchedulingStrategy;
+import org.apache.nifi.util.FlowDifferenceFilters;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.web.FlowModification;
import org.apache.nifi.web.Revision;
@@ -2225,6 +2226,11 @@ public final class DtoFactory {
continue;
}
+ // Ignore differences for adding remote ports
+ if (FlowDifferenceFilters.isAddedRemotePort(difference)) {
+ continue;
+ }
+
final ComponentDifferenceDTO componentDiff = createComponentDifference(difference);
final List<DifferenceDTO> differences = differencesByComponent.computeIfAbsent(componentDiff, key -> new ArrayList<>());
http://git-wip-us.apache.org/repos/asf/nifi/blob/828dd799/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/AffectedComponentUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/AffectedComponentUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/AffectedComponentUtils.java
index f257bb1..05aa4f1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/AffectedComponentUtils.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/AffectedComponentUtils.java
@@ -17,8 +17,6 @@
package org.apache.nifi.web.util;
-import java.util.Optional;
-
import org.apache.nifi.authorization.user.NiFiUser;
import org.apache.nifi.web.NiFiServiceFacade;
import org.apache.nifi.web.api.dto.AffectedComponentDTO;
@@ -31,6 +29,8 @@ import org.apache.nifi.web.api.entity.PortEntity;
import org.apache.nifi.web.api.entity.ProcessorEntity;
import org.apache.nifi.web.api.entity.RemoteProcessGroupEntity;
+import java.util.Optional;
+
public class AffectedComponentUtils {
public static AffectedComponentEntity updateEntity(final AffectedComponentEntity componentEntity, final NiFiServiceFacade serviceFacade,
@@ -81,4 +81,5 @@ public class AffectedComponentUtils {
return null;
}
+
}
[05/50] nifi git commit: NIFI-4768: Add exclusion filters to
S2SProvenanceReportingTask
Posted by al...@apache.org.
NIFI-4768: Add exclusion filters to S2SProvenanceReportingTask
NIFI-4768: Updated exclusion logic per review comments
This closes #2397.
Signed-off-by: Koji Kawamura <ij...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/445f4967
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/445f4967
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/445f4967
Branch: refs/heads/HDF-3.1-maint
Commit: 445f4967c54fb8976adecb340f423ae13313db14
Parents: 82092d6
Author: Matthew Burgess <ma...@apache.org>
Authored: Thu Jan 11 15:00:03 2018 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Tue Jan 16 14:16:51 2018 -0500
----------------------------------------------------------------------
.../provenance/ProvenanceEventConsumer.java | 68 ++++++++++++++---
.../SiteToSiteProvenanceReportingTask.java | 57 ++++++++++++++-
.../TestSiteToSiteProvenanceReportingTask.java | 77 ++++++++++++++++++++
3 files changed, 188 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/445f4967/nifi-nar-bundles/nifi-extension-utils/nifi-reporting-utils/src/main/java/org/apache/nifi/reporting/util/provenance/ProvenanceEventConsumer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-reporting-utils/src/main/java/org/apache/nifi/reporting/util/provenance/ProvenanceEventConsumer.java b/nifi-nar-bundles/nifi-extension-utils/nifi-reporting-utils/src/main/java/org/apache/nifi/reporting/util/provenance/ProvenanceEventConsumer.java
index 18a8afe..3473475 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-reporting-utils/src/main/java/org/apache/nifi/reporting/util/provenance/ProvenanceEventConsumer.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-reporting-utils/src/main/java/org/apache/nifi/reporting/util/provenance/ProvenanceEventConsumer.java
@@ -32,6 +32,7 @@ import org.apache.nifi.reporting.ReportingContext;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -66,8 +67,11 @@ public class ProvenanceEventConsumer {
private String startPositionValue = PROVENANCE_START_POSITION.getDefaultValue();
private Pattern componentTypeRegex;
- private List<ProvenanceEventType> eventTypes = new ArrayList<ProvenanceEventType>();
- private List<String> componentIds = new ArrayList<String>();
+ private Pattern componentTypeRegexExclude;
+ private List<ProvenanceEventType> eventTypes = new ArrayList<>();
+ private List<ProvenanceEventType> eventTypesExclude = new ArrayList<>();
+ private List<String> componentIds = new ArrayList<>();
+ private List<String> componentIdsExclude = new ArrayList<>();
private int batchSize = Integer.parseInt(PROVENANCE_BATCH_SIZE.getDefaultValue());
private volatile long firstEventId = -1L;
@@ -89,16 +93,26 @@ public class ProvenanceEventConsumer {
}
}
- public void addTargetEventType(final ProvenanceEventType... types) {
- for (ProvenanceEventType type : types) {
- eventTypes.add(type);
+ public void setComponentTypeRegexExclude(final String componentTypeRegex) {
+ if (!StringUtils.isBlank(componentTypeRegex)) {
+ this.componentTypeRegexExclude = Pattern.compile(componentTypeRegex);
}
}
+ public void addTargetEventType(final ProvenanceEventType... types) {
+ Collections.addAll(eventTypes, types);
+ }
+
+ public void addTargetEventTypeExclude(final ProvenanceEventType... types) {
+ Collections.addAll(eventTypesExclude, types);
+ }
+
public void addTargetComponentId(final String... ids) {
- for (String id : ids) {
- componentIds.add(id);
- }
+ Collections.addAll(componentIds, ids);
+ }
+
+ public void addTargetComponentIdExclude(final String... ids) {
+ Collections.addAll(componentIdsExclude, ids);
}
public void setScheduled(boolean scheduled) {
@@ -226,7 +240,8 @@ public class ProvenanceEventConsumer {
private boolean isFilteringEnabled() {
- return componentTypeRegex != null || !eventTypes.isEmpty() || !componentIds.isEmpty();
+ return componentTypeRegex != null || !eventTypes.isEmpty() || !componentIds.isEmpty()
+ || componentTypeRegexExclude != null || !eventTypesExclude.isEmpty() || !componentIdsExclude.isEmpty();
}
private List<ProvenanceEventRecord> filterEvents(ComponentMapHolder componentMapHolder, List<ProvenanceEventRecord> provenanceEvents) {
@@ -234,7 +249,38 @@ public class ProvenanceEventConsumer {
List<ProvenanceEventRecord> filteredEvents = new ArrayList<>();
for (ProvenanceEventRecord provenanceEventRecord : provenanceEvents) {
+ if (!eventTypesExclude.isEmpty() && eventTypesExclude.contains(provenanceEventRecord.getEventType())) {
+ continue;
+ }
+ if (!eventTypes.isEmpty() && !eventTypes.contains(provenanceEventRecord.getEventType())) {
+ continue;
+ }
final String componentId = provenanceEventRecord.getComponentId();
+ if (!componentIdsExclude.isEmpty()) {
+ if (componentIdsExclude.contains(componentId)) {
+ continue;
+ }
+ // If we aren't excluding it based on component ID, let's see if this component has a parent process group IDs
+ // that is being excluded
+ if (componentMapHolder == null) {
+ continue;
+ }
+ final String processGroupId = componentMapHolder.getProcessGroupId(componentId, provenanceEventRecord.getComponentType());
+ if (!StringUtils.isEmpty(processGroupId)) {
+
+ // Check if the process group or any parent process group is specified as a target component ID.
+ if (componentIdsExclude.contains(processGroupId)) {
+ continue;
+ }
+ ParentProcessGroupSearchNode parentProcessGroup = componentMapHolder.getProcessGroupParent(processGroupId);
+ while (parentProcessGroup != null && !componentIdsExclude.contains(parentProcessGroup.getId())) {
+ parentProcessGroup = parentProcessGroup.getParent();
+ }
+ if (parentProcessGroup != null) {
+ continue;
+ }
+ }
+ }
if (!componentIds.isEmpty() && !componentIds.contains(componentId)) {
// If we aren't filtering it out based on component ID, let's see if this component has a parent process group IDs
// that is being filtered on
@@ -245,7 +291,6 @@ public class ProvenanceEventConsumer {
if (StringUtils.isEmpty(processGroupId)) {
continue;
}
- // Check if the process group or any parent process group is specified as a target component ID.
if (!componentIds.contains(processGroupId)) {
ParentProcessGroupSearchNode parentProcessGroup = componentMapHolder.getProcessGroupParent(processGroupId);
while (parentProcessGroup != null && !componentIds.contains(parentProcessGroup.getId())) {
@@ -256,7 +301,8 @@ public class ProvenanceEventConsumer {
}
}
}
- if (!eventTypes.isEmpty() && !eventTypes.contains(provenanceEventRecord.getEventType())) {
+
+ if (componentTypeRegexExclude != null && componentTypeRegexExclude.matcher(provenanceEventRecord.getComponentType()).matches()) {
continue;
}
if (componentTypeRegex != null && !componentTypeRegex.matcher(provenanceEventRecord.getComponentType()).matches()) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/445f4967/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
index 61c8bc4..8331b46 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
@@ -87,7 +87,7 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
static final PropertyDescriptor FILTER_EVENT_TYPE = new PropertyDescriptor.Builder()
.name("s2s-prov-task-event-filter")
- .displayName("Event Type")
+ .displayName("Event Type to Include")
.description("Comma-separated list of event types that will be used to filter the provenance events sent by the reporting task. "
+ "Available event types are " + Arrays.deepToString(ProvenanceEventType.values()) + ". If no filter is set, all the events are sent. If "
+ "multiple filters are set, the filters are cumulative.")
@@ -95,24 +95,55 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
+ static final PropertyDescriptor FILTER_EVENT_TYPE_EXCLUDE = new PropertyDescriptor.Builder()
+ .name("s2s-prov-task-event-filter-exclude")
+ .displayName("Event Type to Exclude")
+ .description("Comma-separated list of event types that will be used to exclude the provenance events sent by the reporting task. "
+ + "Available event types are " + Arrays.deepToString(ProvenanceEventType.values()) + ". If no filter is set, all the events are sent. If "
+ + "multiple filters are set, the filters are cumulative. If an event type is included in Event Type to Include and excluded here, then the "
+ + "exclusion takes precedence and the event will not be sent.")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
static final PropertyDescriptor FILTER_COMPONENT_TYPE = new PropertyDescriptor.Builder()
.name("s2s-prov-task-type-filter")
- .displayName("Component Type")
+ .displayName("Component Type to Include")
.description("Regular expression to filter the provenance events based on the component type. Only the events matching the regular "
+ "expression will be sent. If no filter is set, all the events are sent. If multiple filters are set, the filters are cumulative.")
.required(false)
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
.build();
+ static final PropertyDescriptor FILTER_COMPONENT_TYPE_EXCLUDE = new PropertyDescriptor.Builder()
+ .name("s2s-prov-task-type-filter-exclude")
+ .displayName("Component Type to Exclude")
+ .description("Regular expression to exclude the provenance events based on the component type. The events matching the regular "
+ + "expression will not be sent. If no filter is set, all the events are sent. If multiple filters are set, the filters are cumulative. "
+ + "If a component type is included in Component Type to Include and excluded here, then the exclusion takes precedence and the event will not be sent.")
+ .required(false)
+ .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .build();
+
static final PropertyDescriptor FILTER_COMPONENT_ID = new PropertyDescriptor.Builder()
.name("s2s-prov-task-id-filter")
- .displayName("Component ID")
+ .displayName("Component ID to Include")
.description("Comma-separated list of component UUID that will be used to filter the provenance events sent by the reporting task. If no "
+ "filter is set, all the events are sent. If multiple filters are set, the filters are cumulative.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
+ static final PropertyDescriptor FILTER_COMPONENT_ID_EXCLUDE = new PropertyDescriptor.Builder()
+ .name("s2s-prov-task-id-filter-exclude")
+ .displayName("Component ID to Exclude")
+ .description("Comma-separated list of component UUID that will be used to exclude the provenance events sent by the reporting task. If no "
+ + "filter is set, all the events are sent. If multiple filters are set, the filters are cumulative. If a component UUID is included in "
+ + "Component ID to Include and excluded here, then the exclusion takes precedence and the event will not be sent.")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
static final PropertyDescriptor START_POSITION = new PropertyDescriptor.Builder()
.name("start-position")
.displayName("Start Position")
@@ -133,6 +164,7 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
// initialize component type filtering
consumer.setComponentTypeRegex(context.getProperty(FILTER_COMPONENT_TYPE).getValue());
+ consumer.setComponentTypeRegexExclude(context.getProperty(FILTER_COMPONENT_TYPE_EXCLUDE).getValue());
final String[] targetEventTypes = StringUtils.stripAll(StringUtils.split(context.getProperty(FILTER_EVENT_TYPE).getValue(), ','));
if(targetEventTypes != null) {
@@ -145,12 +177,28 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
}
}
+ final String[] targetEventTypesExclude = StringUtils.stripAll(StringUtils.split(context.getProperty(FILTER_EVENT_TYPE_EXCLUDE).getValue(), ','));
+ if(targetEventTypesExclude != null) {
+ for(String type : targetEventTypesExclude) {
+ try {
+ consumer.addTargetEventTypeExclude(ProvenanceEventType.valueOf(type));
+ } catch (Exception e) {
+ getLogger().warn(type + " is not a correct event type, removed from the exclude filtering.");
+ }
+ }
+ }
+
// initialize component ID filtering
final String[] targetComponentIds = StringUtils.stripAll(StringUtils.split(context.getProperty(FILTER_COMPONENT_ID).getValue(), ','));
if(targetComponentIds != null) {
consumer.addTargetComponentId(targetComponentIds);
}
+ final String[] targetComponentIdsExclude = StringUtils.stripAll(StringUtils.split(context.getProperty(FILTER_COMPONENT_ID_EXCLUDE).getValue(), ','));
+ if(targetComponentIdsExclude != null) {
+ consumer.addTargetComponentIdExclude(targetComponentIdsExclude);
+ }
+
consumer.setScheduled(true);
}
@@ -166,8 +214,11 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
final List<PropertyDescriptor> properties = new ArrayList<>(super.getSupportedPropertyDescriptors());
properties.add(PLATFORM);
properties.add(FILTER_EVENT_TYPE);
+ properties.add(FILTER_EVENT_TYPE_EXCLUDE);
properties.add(FILTER_COMPONENT_TYPE);
+ properties.add(FILTER_COMPONENT_TYPE_EXCLUDE);
properties.add(FILTER_COMPONENT_ID);
+ properties.add(FILTER_COMPONENT_ID_EXCLUDE);
properties.add(START_POSITION);
return properties;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/445f4967/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/test/java/org/apache/nifi/reporting/TestSiteToSiteProvenanceReportingTask.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/test/java/org/apache/nifi/reporting/TestSiteToSiteProvenanceReportingTask.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/test/java/org/apache/nifi/reporting/TestSiteToSiteProvenanceReportingTask.java
index 26d5877..31054c2 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/test/java/org/apache/nifi/reporting/TestSiteToSiteProvenanceReportingTask.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/test/java/org/apache/nifi/reporting/TestSiteToSiteProvenanceReportingTask.java
@@ -273,6 +273,25 @@ public class TestSiteToSiteProvenanceReportingTask {
}
@Test
+ public void testFilterComponentTypeExcludeSuccess() throws IOException, InitializationException {
+ final Map<PropertyDescriptor, String> properties = new HashMap<>();
+ for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
+ properties.put(descriptor, descriptor.getDefaultValue());
+ }
+ properties.put(SiteToSiteProvenanceReportingTask.BATCH_SIZE, "1000");
+ properties.put(SiteToSiteProvenanceReportingTask.FILTER_COMPONENT_TYPE_EXCLUDE, "dummy.*");
+
+ ProvenanceEventRecord event = createProvenanceEventRecord();
+
+ MockSiteToSiteProvenanceReportingTask task = setup(event, properties);
+ task.initialize(initContext);
+ task.onScheduled(confContext);
+ task.onTrigger(context);
+
+ assertEquals(0, task.dataSent.size());
+ }
+
+ @Test
public void testFilterComponentTypeNoResult() throws IOException, InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
@@ -292,6 +311,25 @@ public class TestSiteToSiteProvenanceReportingTask {
}
@Test
+ public void testFilterComponentTypeNoResultExcluded() throws IOException, InitializationException {
+ final Map<PropertyDescriptor, String> properties = new HashMap<>();
+ for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
+ properties.put(descriptor, descriptor.getDefaultValue());
+ }
+ properties.put(SiteToSiteProvenanceReportingTask.BATCH_SIZE, "1000");
+ properties.put(SiteToSiteProvenanceReportingTask.FILTER_COMPONENT_TYPE_EXCLUDE, "proc.*");
+
+ ProvenanceEventRecord event = createProvenanceEventRecord();
+
+ MockSiteToSiteProvenanceReportingTask task = setup(event, properties);
+ task.initialize(initContext);
+ task.onScheduled(confContext);
+ task.onTrigger(context);
+
+ assertEquals(3, task.dataSent.size());
+ }
+
+ @Test
public void testFilterEventTypeSuccess() throws IOException, InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
@@ -311,6 +349,25 @@ public class TestSiteToSiteProvenanceReportingTask {
}
@Test
+ public void testFilterEventTypeExcludeSuccess() throws IOException, InitializationException {
+ final Map<PropertyDescriptor, String> properties = new HashMap<>();
+ for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
+ properties.put(descriptor, descriptor.getDefaultValue());
+ }
+ properties.put(SiteToSiteProvenanceReportingTask.BATCH_SIZE, "1000");
+ properties.put(SiteToSiteProvenanceReportingTask.FILTER_EVENT_TYPE_EXCLUDE, "RECEIVE, notExistingType, DROP");
+
+ ProvenanceEventRecord event = createProvenanceEventRecord();
+
+ MockSiteToSiteProvenanceReportingTask task = setup(event, properties);
+ task.initialize(initContext);
+ task.onScheduled(confContext);
+ task.onTrigger(context);
+
+ assertEquals(0, task.dataSent.size());
+ }
+
+ @Test
public void testFilterEventTypeNoResult() throws IOException, InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
@@ -372,6 +429,26 @@ public class TestSiteToSiteProvenanceReportingTask {
}
@Test
+ public void testFilterMultiFilterExcludeTakesPrecedence() throws IOException, InitializationException {
+ final Map<PropertyDescriptor, String> properties = new HashMap<>();
+ for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
+ properties.put(descriptor, descriptor.getDefaultValue());
+ }
+ properties.put(SiteToSiteProvenanceReportingTask.BATCH_SIZE, "1000");
+ properties.put(SiteToSiteProvenanceReportingTask.FILTER_COMPONENT_TYPE_EXCLUDE, "dummy.*");
+ properties.put(SiteToSiteProvenanceReportingTask.FILTER_EVENT_TYPE, "RECEIVE");
+
+ ProvenanceEventRecord event = createProvenanceEventRecord();
+
+ MockSiteToSiteProvenanceReportingTask task = setup(event, properties);
+ task.initialize(initContext);
+ task.onScheduled(confContext);
+ task.onTrigger(context);
+
+ assertEquals(0, task.dataSent.size());
+ }
+
+ @Test
public void testFilterProcessGroupId() throws IOException, InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
for (final PropertyDescriptor descriptor : new MockSiteToSiteProvenanceReportingTask().getSupportedPropertyDescriptors()) {
[37/50] nifi git commit: NIFI-4925: - Addressing memory leak from
lingering authorization results that did not represent actual access
attempts. This closes #2511.
Posted by al...@apache.org.
NIFI-4925:
- Addressing memory leak from lingering authorization results that did not represent actual access attempts. This closes #2511.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/87ec87a6
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/87ec87a6
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/87ec87a6
Branch: refs/heads/HDF-3.1-maint
Commit: 87ec87a677ee878f0a7ae12953b508eb1f5e4a3e
Parents: 2d85d22
Author: Matt Gilman <ma...@gmail.com>
Authored: Fri Mar 2 16:24:34 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Mar 5 15:40:32 2018 -0500
----------------------------------------------------------------------
.../AuthorizationAuditorInvocationHandler.java | 47 ++++++++++++++++++++
.../nifi/authorization/AuthorizerFactory.java | 20 +++++++--
.../authorization/AuthorizerFactoryBean.java | 4 +-
.../authorization/AuthorizerFactoryTest.java | 2 +
.../authorization/RangerNiFiAuthorizer.java | 24 +++++-----
5 files changed, 83 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/87ec87a6/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java
new file mode 100644
index 0000000..7f8d76c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+public class AuthorizationAuditorInvocationHandler implements InvocationHandler {
+
+ private final Authorizer authorizer;
+ private final AuthorizationAuditor auditor;
+
+ public AuthorizationAuditorInvocationHandler(final Authorizer authorizer, final AuthorizationAuditor auditor) {
+ this.authorizer = authorizer;
+ this.auditor = auditor;
+ }
+
+ @Override
+ public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable {
+ try {
+ if (AuthorizationAuditor.class.getMethod("auditAccessAttempt", AuthorizationRequest.class, AuthorizationResult.class).equals(method)) {
+ return method.invoke(auditor, args);
+ } else {
+ return method.invoke(authorizer, args);
+ }
+ } catch (final InvocationTargetException e) {
+ // If the proxied instance throws an Exception, it'll be wrapped in an InvocationTargetException. We want
+ // to instead re-throw what the proxied instance threw, so we pull it out of the InvocationTargetException.
+ throw e.getCause();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/87ec87a6/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java
index 915bff0..812ea17 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java
@@ -101,9 +101,11 @@ public final class AuthorizerFactory {
}
public static Authorizer installIntegrityChecks(final Authorizer baseAuthorizer) {
+ Authorizer authorizer;
+
if (baseAuthorizer instanceof ManagedAuthorizer) {
final ManagedAuthorizer baseManagedAuthorizer = (ManagedAuthorizer) baseAuthorizer;
- return new ManagedAuthorizer() {
+ authorizer = new ManagedAuthorizer() {
@Override
public String getFingerprint() throws AuthorizationAccessException {
return baseManagedAuthorizer.getFingerprint();
@@ -395,7 +397,7 @@ public final class AuthorizerFactory {
}
};
} else {
- return new Authorizer() {
+ authorizer = new Authorizer() {
@Override
public AuthorizationResult authorize(AuthorizationRequest request) throws AuthorizationAccessException {
final AuthorizationResult result = baseAuthorizer.authorize(request);
@@ -422,6 +424,19 @@ public final class AuthorizerFactory {
}
};
}
+
+ // conditionally add support for the audit methods
+ if (baseAuthorizer instanceof AuthorizationAuditor) {
+ final AuthorizationAuditorInvocationHandler invocationHandler = new AuthorizationAuditorInvocationHandler(authorizer, (AuthorizationAuditor) baseAuthorizer);
+
+ final List<Class<?>> interfaceList = ClassUtils.getAllInterfaces(authorizer.getClass());
+ interfaceList.add(AuthorizationAuditor.class);
+ final Class<?>[] interfaces = interfaceList.toArray(new Class<?>[interfaceList.size()]);
+
+ authorizer = (Authorizer) Proxy.newProxyInstance(authorizer.getClass().getClassLoader(), interfaces, invocationHandler);
+ }
+
+ return authorizer;
}
/**
@@ -433,7 +448,6 @@ public final class AuthorizerFactory {
public static Authorizer withNarLoader(final Authorizer baseAuthorizer, final ClassLoader classLoader) {
final AuthorizerInvocationHandler invocationHandler = new AuthorizerInvocationHandler(baseAuthorizer, classLoader);
- // extract all interfaces... baseAuthorizer is non null so getAllInterfaces is non null
final List<Class<?>> interfaceList = ClassUtils.getAllInterfaces(baseAuthorizer.getClass());
final Class<?>[] interfaces = interfaceList.toArray(new Class<?>[interfaceList.size()]);
http://git-wip-us.apache.org/repos/asf/nifi/blob/87ec87a6/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
index 9f17a18..19798a0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
@@ -160,6 +160,8 @@ public class AuthorizerFactoryBean implements FactoryBean, DisposableBean, UserG
// ensure it was found
if (authorizer == null) {
throw new Exception(String.format("The specified authorizer '%s' could not be found.", authorizerIdentifier));
+ } else {
+ authorizer = AuthorizerFactory.installIntegrityChecks(authorizer);
}
}
}
@@ -338,7 +340,7 @@ public class AuthorizerFactoryBean implements FactoryBean, DisposableBean, UserG
authorizerClassLoader = new URLClassLoader(urls, authorizerClassLoader);
}
- return AuthorizerFactory.installIntegrityChecks(AuthorizerFactory.withNarLoader(instance, authorizerClassLoader));
+ return AuthorizerFactory.withNarLoader(instance, authorizerClassLoader);
}
private AuthorizerConfigurationContext loadAuthorizerConfiguration(final String identifier, final List<Property> properties) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/87ec87a6/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java
index caae13f..167a7d4 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java
@@ -289,6 +289,8 @@ public class AuthorizerFactoryTest {
Authorizer authorizer = AuthorizerFactory.installIntegrityChecks(mockAuthorizer);
authorizer.onConfigured(context);
+ assertTrue(authorizer instanceof AuthorizationAuditor);
+
final AuthorizationRequest accessAttempt = new AuthorizationRequest.Builder()
.resource(new MockResource("resource1", "Resource 1"))
.identity("user-1")
http://git-wip-us.apache.org/repos/asf/nifi/blob/87ec87a6/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java
index 8b98d53..a49887b 100644
--- a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java
+++ b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java
@@ -46,9 +46,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.MalformedURLException;
import java.util.Date;
+import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import java.util.WeakHashMap;
/**
* Authorizer implementation that uses Apache Ranger to make authorization decisions.
@@ -71,7 +71,7 @@ public class RangerNiFiAuthorizer implements Authorizer, AuthorizationAuditor {
static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication";
static final String KERBEROS_AUTHENTICATION = "kerberos";
- private final ConcurrentMap<AuthorizationRequest, RangerAccessResult> resultLookup = new ConcurrentHashMap<>();
+ private final Map<AuthorizationRequest, RangerAccessResult> resultLookup = new WeakHashMap<>();
private volatile RangerBasePluginWithPolicies nifiPlugin = null;
private volatile RangerDefaultAuditHandler defaultAuditHandler = null;
@@ -175,10 +175,14 @@ public class RangerNiFiAuthorizer implements Authorizer, AuthorizationAuditor {
final RangerAccessResult result = nifiPlugin.isAccessAllowed(rangerRequest);
- if (result != null && result.getIsAllowed()) {
- // store the result for auditing purposes later
- resultLookup.put(request, result);
+ // store the result for auditing purposes later if appropriate
+ if (request.isAccessAttempt()) {
+ synchronized (resultLookup) {
+ resultLookup.put(request, result);
+ }
+ }
+ if (result != null && result.getIsAllowed()) {
// return approved
return AuthorizationResult.approved();
} else {
@@ -192,9 +196,6 @@ public class RangerNiFiAuthorizer implements Authorizer, AuthorizationAuditor {
logger.debug(String.format("Unable to authorize %s due to %s", identity, reason));
}
- // store the result for auditing purposes later
- resultLookup.put(request, result);
-
// a policy does exist for the resource so we were really denied access here
return AuthorizationResult.denied(request.getExplanationSupplier().get());
} else {
@@ -206,7 +207,10 @@ public class RangerNiFiAuthorizer implements Authorizer, AuthorizationAuditor {
@Override
public void auditAccessAttempt(final AuthorizationRequest request, final AuthorizationResult result) {
- final RangerAccessResult rangerResult = resultLookup.remove(request);
+ final RangerAccessResult rangerResult;
+ synchronized (resultLookup) {
+ rangerResult = resultLookup.remove(request);
+ }
if (rangerResult != null && rangerResult.getIsAudited()) {
AuthzAuditEvent event = defaultAuditHandler.getAuthzEvents(rangerResult);
[29/50] nifi git commit: NIFI-4837: - When Jetty initializes fails,
performing a shutdown sequence to ensure all allocated resources are
released.
Posted by al...@apache.org.
NIFI-4837:
- When Jetty initializes fails, performing a shutdown sequence to ensure all allocated resources are released.
This closes #2455.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9a59d9b8
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9a59d9b8
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9a59d9b8
Branch: refs/heads/HDF-3.1-maint
Commit: 9a59d9b8fcd8a8f8574be348491f7c3813965ba6
Parents: 63098ba
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Feb 7 13:52:02 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 20:22:15 2018 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/HandleHttpRequest.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9a59d9b8/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index 5f17156..d436312 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -485,7 +485,15 @@ public class HandleHttpRequest extends AbstractProcessor {
}
} catch (Exception e) {
context.yield();
- throw new ProcessException("Failed to initialize the server",e);
+
+ try {
+ // shutdown to release any resources allocated during the failed initialization
+ shutdown();
+ } catch (final Exception shutdownException) {
+ getLogger().debug("Failed to shutdown following a failed initialization: " + shutdownException);
+ }
+
+ throw new ProcessException("Failed to initialize the server", e);
}
HttpRequestContainer container;
[50/50] nifi git commit: MINIFI-4788 Exposing nifi.web.proxy.host
configuration for Docker containers.
Posted by al...@apache.org.
MINIFI-4788 Exposing nifi.web.proxy.host configuration for Docker containers.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/29b82727
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/29b82727
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/29b82727
Branch: refs/heads/HDF-3.1-maint
Commit: 29b82727a10ba44dda2acc545056d78db126901c
Parents: aa4384b
Author: Aldrin Piri <al...@apache.org>
Authored: Tue Jan 23 10:43:45 2018 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Thu May 10 11:49:14 2018 -0400
----------------------------------------------------------------------
nifi-docker/dockerhub/README.md | 3 +++
nifi-docker/dockerhub/sh/secure.sh | 7 +++++++
nifi-docker/dockerhub/sh/start.sh | 5 +++++
3 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/29b82727/nifi-docker/dockerhub/README.md
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/README.md b/nifi-docker/dockerhub/README.md
index d87765e..2998aa0 100644
--- a/nifi-docker/dockerhub/README.md
+++ b/nifi-docker/dockerhub/README.md
@@ -123,3 +123,6 @@ can be published to the host.
| HTTP Port | nifi.web.http.port | 8080 |
| HTTPS Port | nifi.web.https.port | 8443 |
| Remote Input Socket Port | nifi.remote.input.socket.port | 10000 |
+
+**NOTE**: If mapping the HTTPS port specifying trusted hosts should be provided for the property _nifi.web.proxy.host_. This property can be specified to running instances
+via specifying an environment variable at container instantiation of _NIFI\_WEB\_PROXY\_HOST_.
http://git-wip-us.apache.org/repos/asf/nifi/blob/29b82727/nifi-docker/dockerhub/sh/secure.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/sh/secure.sh b/nifi-docker/dockerhub/sh/secure.sh
index 4609005..2de53c6 100644
--- a/nifi-docker/dockerhub/sh/secure.sh
+++ b/nifi-docker/dockerhub/sh/secure.sh
@@ -52,6 +52,13 @@ prop_replace 'nifi.web.https.port' '8443'
prop_replace 'nifi.web.https.host' "${hostname}"
prop_replace 'nifi.remote.input.secure' 'true'
+# Check if the user has specified a nifi.web.proxy.host setting and handle appropriately
+if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then
+ echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping.'
+else
+ prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}"
+fi
+
# Establish initial user and an associated admin identity
sed -i -e 's|<property name="Initial User Identity 1"></property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml
sed -i -e 's|<property name="Initial Admin Identity"></property>|<property name="Initial Admin Identity">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml
http://git-wip-us.apache.org/repos/asf/nifi/blob/29b82727/nifi-docker/dockerhub/sh/start.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/sh/start.sh b/nifi-docker/dockerhub/sh/start.sh
index df79ad6..d1c2754 100755
--- a/nifi-docker/dockerhub/sh/start.sh
+++ b/nifi-docker/dockerhub/sh/start.sh
@@ -41,6 +41,11 @@ case ${AUTH} in
. "${scripts_dir}/secure.sh"
. "${scripts_dir}/update_login_providers.sh"
;;
+ *)
+ if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then
+ echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.'
+ fi
+ ;;
esac
# Continuously provide logs so that 'docker logs' can produce them
[35/50] nifi git commit: NIFI-4863: Bug fixes to the way that we
handled Remote Group Ports when changing flow version / reverting local
changes - Everywhere that we ignore adding remote ports we should ignore
removing remote ports as well in flow diffs
Posted by al...@apache.org.
NIFI-4863: Bug fixes to the way that we handled Remote Group Ports when changing flow version / reverting local changes
- Everywhere that we ignore adding remote ports we should ignore removing remote ports as well in flow diffs
This closes #2462.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/33c44b5f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/33c44b5f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/33c44b5f
Branch: refs/heads/HDF-3.1-maint
Commit: 33c44b5fe84db2dac6a31a76cbae83715d66a50a
Parents: 828dd79
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Feb 9 14:09:23 2018 -0500
Committer: Bryan Bende <bb...@apache.org>
Committed: Fri Feb 9 15:47:43 2018 -0500
----------------------------------------------------------------------
.../nifi/groups/StandardProcessGroup.java | 27 ++++---
.../nifi/remote/StandardRemoteProcessGroup.java | 1 +
.../apache/nifi/util/FlowDifferenceFilters.java | 8 +--
.../nifi/util/TestFlowDifferenceFilters.java | 8 +--
.../org/apache/nifi/web/NiFiServiceFacade.java | 12 ++++
.../nifi/web/StandardNiFiServiceFacade.java | 40 +++++++++--
.../apache/nifi/web/api/VersionsResource.java | 74 ++++++++++++++------
.../org/apache/nifi/web/api/dto/DtoFactory.java | 3 +-
.../web/dao/impl/StandardProcessGroupDAO.java | 14 ++--
.../dao/impl/StandardRemoteProcessGroupDAO.java | 14 ++--
10 files changed, 138 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index 8b7dcd2..ae9b43f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -3270,11 +3270,6 @@ public final class StandardProcessGroup implements ProcessGroup {
continue;
}
- // Ignore differences for adding a remote port
- if (FlowDifferenceFilters.isAddedRemotePort(diff)) {
- continue;
- }
-
// If this update adds a new Controller Service, then we need to check if the service already exists at a higher level
// and if so compare our VersionedControllerService to the existing service.
if (diff.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
@@ -3909,9 +3904,17 @@ public final class StandardProcessGroup implements ProcessGroup {
}
final RemoteProcessGroup rpg = rpgOption.get();
- return rpg.getInputPorts().stream()
+ final Optional<RemoteGroupPort> portByIdOption = rpg.getInputPorts().stream()
.filter(component -> component.getVersionedComponentId().isPresent())
.filter(component -> id.equals(component.getVersionedComponentId().get()))
+ .findAny();
+
+ if (portByIdOption.isPresent()) {
+ return portByIdOption.get();
+ }
+
+ return rpg.getInputPorts().stream()
+ .filter(component -> connectableComponent.getName().equals(component.getName()))
.findAny()
.orElse(null);
}
@@ -3928,9 +3931,17 @@ public final class StandardProcessGroup implements ProcessGroup {
}
final RemoteProcessGroup rpg = rpgOption.get();
- return rpg.getOutputPorts().stream()
+ final Optional<RemoteGroupPort> portByIdOption = rpg.getOutputPorts().stream()
.filter(component -> component.getVersionedComponentId().isPresent())
.filter(component -> id.equals(component.getVersionedComponentId().get()))
+ .findAny();
+
+ if (portByIdOption.isPresent()) {
+ return portByIdOption.get();
+ }
+
+ return rpg.getOutputPorts().stream()
+ .filter(component -> connectableComponent.getName().equals(component.getName()))
.findAny()
.orElse(null);
}
@@ -4216,7 +4227,7 @@ public final class StandardProcessGroup implements ProcessGroup {
final FlowComparison comparison = flowComparator.compare();
final Set<FlowDifference> differences = comparison.getDifferences().stream()
.filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
- .filter(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS)
+ .filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS)
.collect(Collectors.toCollection(HashSet::new));
LOG.debug("There are {} differences between this Local Flow and the Versioned Flow: {}", differences.size(), differences);
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
index 0b9c6f2..c4621e6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
@@ -672,6 +672,7 @@ public class StandardRemoteProcessGroup implements RemoteProcessGroup {
if (!StringUtils.isBlank(descriptor.getBatchDuration())) {
port.setBatchDuration(descriptor.getBatchDuration());
}
+ port.setVersionedComponentId(descriptor.getVersionedComponentId());
return port;
} finally {
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
index ca48b99..4d341e9 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowDifferenceFilters.java
@@ -29,12 +29,12 @@ public class FlowDifferenceFilters {
/**
* Predicate that returns true if the difference is NOT a remote port being added, and false if it is.
*/
- public static Predicate<FlowDifference> FILTER_ADDED_REMOTE_PORTS = (fd) -> {
- return !isAddedRemotePort(fd);
+ public static Predicate<FlowDifference> FILTER_ADDED_REMOVED_REMOTE_PORTS = (fd) -> {
+ return !isAddedOrRemovedRemotePort(fd);
};
- public static boolean isAddedRemotePort(final FlowDifference fd) {
- if (fd.getDifferenceType() == DifferenceType.COMPONENT_ADDED) {
+ public static boolean isAddedOrRemovedRemotePort(final FlowDifference fd) {
+ if (fd.getDifferenceType() == DifferenceType.COMPONENT_ADDED || fd.getDifferenceType() == DifferenceType.COMPONENT_REMOVED) {
VersionedComponent component = fd.getComponentA();
if (component == null || fd.getComponentB() instanceof InstantiatedVersionedComponent) {
component = fd.getComponentB();
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
index ee65816..9cc8b48 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/util/TestFlowDifferenceFilters.java
@@ -35,7 +35,7 @@ public class TestFlowDifferenceFilters {
DifferenceType.COMPONENT_ADDED, null, remoteGroupPort, null, null, "");
// predicate should return false because we don't want to include changes for adding a remote input port
- Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS.test(flowDifference));
}
@Test
@@ -47,7 +47,7 @@ public class TestFlowDifferenceFilters {
DifferenceType.COMPONENT_ADDED, remoteGroupPort, null, null, null, "");
// predicate should return false because we don't want to include changes for adding a remote input port
- Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS.test(flowDifference));
}
@Test
@@ -59,7 +59,7 @@ public class TestFlowDifferenceFilters {
DifferenceType.COMPONENT_ADDED, null, remoteGroupPort, null, null, "");
// predicate should return false because we don't want to include changes for adding a remote input port
- Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ Assert.assertFalse(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS.test(flowDifference));
}
@Test
@@ -71,6 +71,6 @@ public class TestFlowDifferenceFilters {
DifferenceType.COMPONENT_ADDED, null, versionedProcessor, null, null, "");
// predicate should return true because we do want to include changes for adding a non-port
- Assert.assertTrue(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS.test(flowDifference));
+ Assert.assertTrue(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS.test(flowDifference));
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index 165af45..009b096 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -1118,6 +1118,18 @@ public interface NiFiServiceFacade {
RemoteProcessGroupEntity createRemoteProcessGroup(Revision revision, String groupId, RemoteProcessGroupDTO remoteProcessGroupDTO);
/**
+ * Determines whether or not the Remote Port with the given remotePortId is connected (i.e., there are incoming Connections if it's an Input Port
+ * or there are outgoing Connections if it's an Output Port).
+ *
+ * @param remoteProcessGroupId the ID of the Remote Process Group
+ * @param remotePortId the ID of the Port
+ * @return <code>true</code> if remote port identified is connected, <code>false</code> if the port is not connected
+ *
+ * @throws ResourceNotFoundException if the port cannot be found
+ */
+ boolean isRemoteGroupPortConnected(String remoteProcessGroupId, String remotePortId);
+
+ /**
* Gets a remote process group.
*
* @param remoteProcessGroupId The id of the remote process group
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index bf794cf..f57f628 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -1854,6 +1854,22 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
+ public boolean isRemoteGroupPortConnected(final String remoteProcessGroupId, final String remotePortId) {
+ final RemoteProcessGroup rpg = remoteProcessGroupDAO.getRemoteProcessGroup(remoteProcessGroupId);
+ RemoteGroupPort port = rpg.getInputPort(remotePortId);
+ if (port != null) {
+ return port.hasIncomingConnection();
+ }
+
+ port = rpg.getOutputPort(remotePortId);
+ if (port != null) {
+ return !port.getConnections().isEmpty();
+ }
+
+ throw new ResourceNotFoundException("Could not find Port with ID " + remotePortId + " as a child of RemoteProcessGroup with ID " + remoteProcessGroupId);
+ }
+
+ @Override
public void verifyCanAddTemplate(String groupId, String name) {
templateDAO.verifyCanAddTemplate(name, groupId);
}
@@ -3966,7 +3982,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream()
.filter(difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED) // components that are added are not components that will be affected in the local flow.
.filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
- .filter(FlowDifferenceFilters.FILTER_ADDED_REMOTE_PORTS)
+ .filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS)
.map(difference -> {
final VersionedComponent localComponent = difference.getComponentA();
@@ -4004,7 +4020,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
// Ignore differences for adding remote ports
- if (FlowDifferenceFilters.isAddedRemotePort(difference)) {
+ if (FlowDifferenceFilters.isAddedOrRemovedRemotePort(difference)) {
continue;
}
@@ -4105,12 +4121,20 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
private void mapToConnectableId(final Collection<? extends Connectable> connectables, final Map<String, List<Connectable>> destination) {
for (final Connectable connectable : connectables) {
- final Optional<String> versionedId = connectable.getVersionedComponentId();
- if (!versionedId.isPresent()) {
- continue;
+ final Optional<String> versionedIdOption = connectable.getVersionedComponentId();
+
+ // Determine the Versioned ID by using the ID that is assigned, if one is. Otherwise,
+ // we will calculate the Versioned ID. This allows us to map connectables that currently are not under
+ // version control. We have to do this so that if we are changing flow versions and have a component that is running and it does not exist
+ // in the Versioned Flow, we still need to be able to create an AffectedComponentDTO for it.
+ final String versionedId;
+ if (versionedIdOption.isPresent()) {
+ versionedId = versionedIdOption.get();
+ } else {
+ versionedId = UUID.nameUUIDFromBytes(connectable.getIdentifier().getBytes(StandardCharsets.UTF_8)).toString();
}
- final List<Connectable> byVersionedId = destination.computeIfAbsent(versionedId.get(), key -> new ArrayList<>());
+ final List<Connectable> byVersionedId = destination.computeIfAbsent(versionedId, key -> new ArrayList<>());
byVersionedId.add(connectable);
}
}
@@ -4128,9 +4152,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
final AffectedComponentDTO dto = new AffectedComponentDTO();
dto.setId(connectable.getIdentifier());
dto.setReferenceType(connectable.getConnectableType().name());
- dto.setProcessGroupId(connectable.getProcessGroupIdentifier());
dto.setState(connectable.getScheduledState().name());
+ final String groupId = connectable instanceof RemoteGroupPort ? ((RemoteGroupPort) connectable).getRemoteProcessGroup().getIdentifier() : connectable.getProcessGroupIdentifier();
+ dto.setProcessGroupId(groupId);
+
entity.setComponent(dto);
return entity;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
index 8028bd9..7a96ebf 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java
@@ -17,12 +17,22 @@
package org.apache.nifi.web.api;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import io.swagger.annotations.ApiParam;
-import io.swagger.annotations.ApiResponse;
-import io.swagger.annotations.ApiResponses;
-import io.swagger.annotations.Authorization;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.authorization.AccessDeniedException;
import org.apache.nifi.authorization.Authorizer;
@@ -75,21 +85,6 @@ import org.apache.nifi.web.util.LifecycleManagementException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.MultivaluedHashMap;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -105,6 +100,13 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.stream.Collectors;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import io.swagger.annotations.Authorization;
+
@Path("/versions")
@Api(value = "/versions", description = "Endpoint for managing version control for a flow")
public class VersionsResource extends ApplicationResource {
@@ -1491,6 +1493,36 @@ public class VersionsResource extends ApplicationResource {
// Step 14. Restart all components
final Set<AffectedComponentEntity> componentsToStart = getUpdatedEntities(runningComponents, user);
+
+ // If there are any Remote Group Ports that are supposed to be started and have no connections, we want to remove those from our Set.
+ // This will happen if the Remote Group Port is transmitting when the version change happens but the new flow version does not have
+ // a connection to the port. In such a case, the Port still is included in the Updated Entities because we do not remove them
+ // when updating the flow (they are removed in the background).
+ final Set<AffectedComponentEntity> avoidStarting = new HashSet<>();
+ for (final AffectedComponentEntity componentEntity : componentsToStart) {
+ final AffectedComponentDTO componentDto = componentEntity.getComponent();
+ final String referenceType = componentDto.getReferenceType();
+ if (!AffectedComponentDTO.COMPONENT_TYPE_REMOTE_INPUT_PORT.equals(referenceType)
+ && !AffectedComponentDTO.COMPONENT_TYPE_REMOTE_OUTPUT_PORT.equals(referenceType)) {
+ continue;
+ }
+
+ boolean startComponent;
+ try {
+ startComponent = serviceFacade.isRemoteGroupPortConnected(componentDto.getProcessGroupId(), componentDto.getId());
+ } catch (final ResourceNotFoundException rnfe) {
+ // Could occur if RPG is refreshed at just the right time.
+ startComponent = false;
+ }
+
+ // We must add the components to avoid starting to a separate Set and then remove them below,
+ // rather than removing the component here, because doing so would result in a ConcurrentModificationException.
+ if (!startComponent) {
+ avoidStarting.add(componentEntity);
+ }
+ }
+ componentsToStart.removeAll(avoidStarting);
+
final CancellableTimedPause startComponentsPause = new CancellableTimedPause(250, Long.MAX_VALUE, TimeUnit.MILLISECONDS);
asyncRequest.setCancelCallback(startComponentsPause::cancel);
logger.info("Restarting {} Processors", componentsToStart.size());
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 2ac31fb..359b524 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -1535,6 +1535,7 @@ public final class DtoFactory {
final RemoteProcessGroupPortDTO dto = new RemoteProcessGroupPortDTO();
dto.setId(port.getIdentifier());
+ dto.setGroupId(port.getRemoteProcessGroup().getIdentifier());
dto.setTargetId(port.getTargetIdentifier());
dto.setName(port.getName());
dto.setComments(port.getComments());
@@ -2227,7 +2228,7 @@ public final class DtoFactory {
}
// Ignore differences for adding remote ports
- if (FlowDifferenceFilters.isAddedRemotePort(difference)) {
+ if (FlowDifferenceFilters.isAddedOrRemovedRemotePort(difference)) {
continue;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java
index e1d9e69..4744324 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java
@@ -167,12 +167,9 @@ public class StandardProcessGroupDAO extends ComponentDAO implements ProcessGrou
connectable.getProcessGroup().startOutputPort((Port) connectable);
break;
case REMOTE_INPUT_PORT:
- final RemoteGroupPort remoteInputPort = group.findRemoteGroupPort(componentId);
- remoteInputPort.getRemoteProcessGroup().startTransmitting(remoteInputPort);
- break;
case REMOTE_OUTPUT_PORT:
- final RemoteGroupPort remoteOutputPort = group.findRemoteGroupPort(componentId);
- remoteOutputPort.getRemoteProcessGroup().startTransmitting(remoteOutputPort);
+ final RemoteGroupPort remotePort = group.findRemoteGroupPort(componentId);
+ remotePort.getRemoteProcessGroup().startTransmitting(remotePort);
break;
}
} else {
@@ -188,12 +185,9 @@ public class StandardProcessGroupDAO extends ComponentDAO implements ProcessGrou
connectable.getProcessGroup().stopOutputPort((Port) connectable);
break;
case REMOTE_INPUT_PORT:
- final RemoteGroupPort remoteInputPort = group.findRemoteGroupPort(componentId);
- remoteInputPort.getRemoteProcessGroup().stopTransmitting(remoteInputPort);
- break;
case REMOTE_OUTPUT_PORT:
- final RemoteGroupPort remoteOutputPort = group.findRemoteGroupPort(componentId);
- remoteOutputPort.getRemoteProcessGroup().stopTransmitting(remoteOutputPort);
+ final RemoteGroupPort remotePort = group.findRemoteGroupPort(componentId);
+ remotePort.getRemoteProcessGroup().stopTransmitting(remotePort);
break;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/33c44b5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java
index 8228a77..05a983c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java
@@ -18,11 +18,6 @@ package org.apache.nifi.web.dao.impl;
import static org.apache.nifi.util.StringUtils.isEmpty;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Matcher;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.controller.FlowController;
@@ -39,12 +34,14 @@ import org.apache.nifi.web.api.dto.DtoFactory;
import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO;
import org.apache.nifi.web.dao.RemoteProcessGroupDAO;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
public class StandardRemoteProcessGroupDAO extends ComponentDAO implements RemoteProcessGroupDAO {
- private static final Logger logger = LoggerFactory.getLogger(StandardRemoteProcessGroupDAO.class);
private FlowController flowController;
private RemoteProcessGroup locateRemoteProcessGroup(final String remoteProcessGroupId) {
@@ -129,6 +126,7 @@ public class StandardRemoteProcessGroupDAO extends ComponentDAO implements Remot
/**
* Verifies the specified remote group can be updated, if necessary.
*/
+ @SuppressWarnings("unchecked")
private void verifyUpdate(RemoteProcessGroup remoteProcessGroup, RemoteProcessGroupDTO remoteProcessGroupDto) {
// see if the remote process group can start/stop transmitting
if (isNotNull(remoteProcessGroupDto.isTransmitting())) {
[19/50] nifi git commit: BUG-86293: Update hive dependencies
Posted by al...@apache.org.
BUG-86293: Update hive dependencies
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9d7cdb69
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9d7cdb69
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9d7cdb69
Branch: refs/heads/HDF-3.1-maint
Commit: 9d7cdb69353032fa53e24131e7be91770b8245c9
Parents: d948712
Author: Matthew Burgess <ma...@apache.org>
Authored: Fri Jan 19 18:28:32 2018 -0500
Committer: Matthew Burgess <ma...@apache.org>
Committed: Fri Jan 19 18:28:32 2018 -0500
----------------------------------------------------------------------
pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9d7cdb69/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fb48610..fc69e0b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,8 +103,8 @@
<hadoop.http.core.version>4.2.5</hadoop.http.core.version>
<yammer.metrics.version>2.2.0</yammer.metrics.version>
<ranger.version>0.7.1</ranger.version>
- <hive.version>1.2.1</hive.version>
- <hive.hadoop.version>2.6.2</hive.hadoop.version>
+ <hive.version>1.2.1000.2.6.0.3-8</hive.version>
+ <hive.hadoop.version>2.7.3.2.6.0.3-8</hive.hadoop.version>
<hbase.version>1.1.2</hbase.version>
<storm.version>1.1.1</storm.version>
<hwx.registry.version>0.3.0</hwx.registry.version>
[18/50] nifi git commit: NIFI-4761 Fixing test case regression.
Posted by al...@apache.org.
NIFI-4761 Fixing test case regression.
This closes #2418.
Signed-off-by: Andy LoPresto <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d9487129
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d9487129
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d9487129
Branch: refs/heads/HDF-3.1-maint
Commit: d9487129cc3abb94a6863f81c23c92e02bc1c870
Parents: c582d86
Author: Matt Gilman <ma...@gmail.com>
Authored: Fri Jan 19 12:42:47 2018 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Jan 19 12:49:22 2018 -0500
----------------------------------------------------------------------
.../apache/nifi/web/server/HostHeaderHandler.java | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d9487129/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
index 9a07fcb..72e8d84 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
@@ -70,15 +70,20 @@ public class HostHeaderHandler extends ScopedHandler {
this.serverName = Objects.requireNonNull(serverName);
this.serverPort = serverPort;
- // Default values across generic instances
validHosts = generateDefaultHostnames(null);
-
- // specified host:port
validHosts.add(serverName.toLowerCase());
validHosts.add(serverName.toLowerCase() + ":" + serverPort);
-
- // empty is ok here
+ // Sometimes the hostname is left empty but the port is always populated
+ validHosts.add("localhost");
+ validHosts.add("localhost:" + serverPort);
+ // Different from customizer -- empty is ok here
validHosts.add("");
+ try {
+ validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase());
+ validHosts.add(InetAddress.getLocalHost().getHostName().toLowerCase() + ":" + serverPort);
+ } catch (final Exception e) {
+ logger.warn("Failed to determine local hostname.", e);
+ }
logger.info("Created " + this.toString());
}
[30/50] nifi git commit: NIFI-4844: Adjust BigDecimal scale to the
target Avro schema
Posted by al...@apache.org.
NIFI-4844: Adjust BigDecimal scale to the target Avro schema
- Applied the same scale adjustment not only to BigDecimal inputs, but
also to Double values.
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2450
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6d844783
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6d844783
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6d844783
Branch: refs/heads/HDF-3.1-maint
Commit: 6d844783668090d2d318708aac8c9155c380566b
Parents: 9a59d9b
Author: Koji Kawamura <ij...@apache.org>
Authored: Tue Feb 6 13:52:48 2018 +0900
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Feb 7 20:23:09 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 17 ++++----
.../org/apache/nifi/avro/TestAvroTypeUtil.java | 44 ++++++++++++++++++++
2 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/6d844783/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index 48f661f..f7b8d9b 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -19,7 +19,6 @@ package org.apache.nifi.avro;
import java.io.IOException;
import java.math.BigDecimal;
-import java.math.MathContext;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
@@ -472,7 +471,7 @@ public class AvroTypeUtil {
/**
* Convert a raw value to an Avro object to serialize in Avro type system.
- * The counter-part method which reads an Avro object back to a raw value is {@link #normalizeValue(Object, Schema)}.
+ * The counter-part method which reads an Avro object back to a raw value is {@link #normalizeValue(Object, Schema, String)}.
*/
public static Object convertToAvroObject(final Object rawValue, final Schema fieldSchema) {
return convertToAvroObject(rawValue, fieldSchema, fieldSchema.getName());
@@ -531,18 +530,18 @@ public class AvroTypeUtil {
final LogicalType logicalType = fieldSchema.getLogicalType();
if (logicalType != null && LOGICAL_TYPE_DECIMAL.equals(logicalType.getName())) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
- final BigDecimal decimal;
+ final BigDecimal rawDecimal;
if (rawValue instanceof BigDecimal) {
- final BigDecimal rawDecimal = (BigDecimal) rawValue;
- final int desiredScale = decimalType.getScale();
- // If the desired scale is different than this value's coerce scale.
- decimal = rawDecimal.scale() == desiredScale ? rawDecimal : rawDecimal.setScale(desiredScale, BigDecimal.ROUND_HALF_UP);
+ rawDecimal = (BigDecimal) rawValue;
} else if (rawValue instanceof Double) {
- // Scale is adjusted based on precision. If double was 123.456 and precision is 5, then decimal would be 123.46.
- decimal = new BigDecimal((Double) rawValue, new MathContext(decimalType.getPrecision()));
+ rawDecimal = BigDecimal.valueOf((Double) rawValue);
} else {
throw new IllegalTypeConversionException("Cannot convert value " + rawValue + " of type " + rawValue.getClass() + " to a logical decimal");
}
+ // If the desired scale is different than this value's coerce scale.
+ final int desiredScale = decimalType.getScale();
+ final BigDecimal decimal = rawDecimal.scale() == desiredScale
+ ? rawDecimal : rawDecimal.setScale(desiredScale, BigDecimal.ROUND_HALF_UP);
return new Conversions.DecimalConversion().toBytes(decimal, fieldSchema, logicalType);
}
if (rawValue instanceof byte[]) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/6d844783/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
index cf096aa..d644f7c 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
@@ -21,11 +21,17 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import org.apache.avro.Conversions;
+import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
@@ -259,4 +265,42 @@ public class TestAvroTypeUtil {
}
}
+ @Test
+ public void testToDecimalConversion() {
+ final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
+ final Schema fieldSchema = Schema.create(Type.BYTES);
+ decimalType.addToSchema(fieldSchema);
+
+ final Map<Object, String> expects = new HashMap<>();
+
+ // Double to Decimal
+ expects.put(123d, "123.00000000");
+ // Double can not represent exact 1234567890.12345678, so use 1 less digit to test here.
+ expects.put(1234567890.12345678d, "1234567890.12345670");
+ expects.put(123456789.12345678d, "123456789.12345678");
+ expects.put(1234567890123456d, "1234567890123456.00000000");
+ // ROUND HALF UP.
+ expects.put(0.1234567890123456d, "0.12345679");
+
+
+ // BigDecimal to BigDecimal
+ expects.put(new BigDecimal("123"), "123.00000000");
+ expects.put(new BigDecimal("1234567890.12345678"), "1234567890.12345678");
+ expects.put(new BigDecimal("123456789012345678"), "123456789012345678.00000000");
+ // ROUND HALF UP.
+ expects.put(new BigDecimal("0.123456789012345678"), "0.12345679");
+
+
+ expects.forEach((rawValue, expect) -> {
+ final Object convertedValue = AvroTypeUtil.convertToAvroObject(rawValue, fieldSchema);
+
+ assertTrue(convertedValue instanceof ByteBuffer);
+ final ByteBuffer serializedBytes = (ByteBuffer) convertedValue;
+
+ final BigDecimal bigDecimal = new Conversions.DecimalConversion().fromBytes(serializedBytes, fieldSchema, decimalType);
+ assertEquals(String.format("%s should be converted to %s", rawValue, expect), expect, bigDecimal.toString());
+ });
+
+ }
+
}
[22/50] nifi git commit: NIFI-4803: Ensure that we set the
FileCountAttribute of BinManager to the proper value,
regardless of the merge strategy
Posted by al...@apache.org.
NIFI-4803: Ensure that we set the FileCountAttribute of BinManager to the proper value, regardless of the merge strategy
This closes #2423.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d5f3eef7
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d5f3eef7
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d5f3eef7
Branch: refs/heads/HDF-3.1-maint
Commit: d5f3eef7c25ecbd06c76855ffc3b6ea81bda1dc3
Parents: a261a0d
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Jan 22 14:50:19 2018 -0500
Committer: Bryan Bende <bb...@apache.org>
Committed: Mon Jan 22 15:41:21 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/processors/standard/MergeContent.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d5f3eef7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
index e78cd4f..5b8007b 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
@@ -426,6 +426,8 @@ public class MergeContent extends BinFiles {
protected void setUpBinManager(final BinManager binManager, final ProcessContext context) {
if (MERGE_STRATEGY_DEFRAGMENT.equals(context.getProperty(MERGE_STRATEGY).getValue())) {
binManager.setFileCountAttribute(FRAGMENT_COUNT_ATTRIBUTE);
+ } else {
+ binManager.setFileCountAttribute(null);
}
}