You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2017/02/14 15:20:55 UTC
nifi git commit: NIFI-3362 update FlowConfiguration.xsd to allow all
current time period units
Repository: nifi
Updated Branches:
refs/heads/master bd3e0438a -> 690130b06
NIFI-3362 update FlowConfiguration.xsd to allow all current time period units
This closes #1501.
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/690130b0
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/690130b0
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/690130b0
Branch: refs/heads/master
Commit: 690130b063619adc8ead954afcd62e8dc7db02df
Parents: bd3e043
Author: Mike Moser <mo...@apache.org>
Authored: Fri Feb 10 20:06:16 2017 +0000
Committer: Aldrin Piri <al...@apache.org>
Committed: Tue Feb 14 10:20:39 2017 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/util/FormatUtils.java | 4 +-
.../nifi/fingerprint/FingerprintFactory.java | 7 +-
.../src/main/resources/FlowConfiguration.xsd | 2 +-
.../fingerprint/FingerprintFactoryTest.java | 56 ++++++-
.../nifi/fingerprint/validating-flow.xml | 157 +++++++++++++++++++
5 files changed, 218 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/690130b0/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
index cb9ab36..e129993 100644
--- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
@@ -35,8 +35,8 @@ public class FormatUtils {
private static final double BYTES_IN_TERABYTE = BYTES_IN_GIGABYTE * 1024;
// for Time Durations
- private static final String NANOS = join(UNION, "ns", "nano", "nanos", "nanoseconds");
- private static final String MILLIS = join(UNION, "ms", "milli", "millis", "milliseconds");
+ private static final String NANOS = join(UNION, "ns", "nano", "nanos", "nanosecond", "nanoseconds");
+ private static final String MILLIS = join(UNION, "ms", "milli", "millis", "millisecond", "milliseconds");
private static final String SECS = join(UNION, "s", "sec", "secs", "second", "seconds");
private static final String MINS = join(UNION, "m", "min", "mins", "minute", "minutes");
private static final String HOURS = join(UNION, "h", "hr", "hrs", "hour", "hours");
http://git-wip-us.apache.org/repos/asf/nifi/blob/690130b0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java
index c4da7b6..3634850 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java
@@ -87,7 +87,7 @@ public class FingerprintFactory {
// no fingerprint value
public static final String NO_VALUE = "NO_VALUE";
- private static final String FLOW_CONFIG_XSD = "/FlowConfiguration.xsd";
+ static final String FLOW_CONFIG_XSD = "/FlowConfiguration.xsd";
private static final String ENCRYPTED_VALUE_PREFIX = "enc{";
private static final String ENCRYPTED_VALUE_SUFFIX = "}";
private final StringEncryptor encryptor;
@@ -115,6 +115,11 @@ public class FingerprintFactory {
}
}
+ public FingerprintFactory(final StringEncryptor encryptor, final DocumentBuilder docBuilder) {
+ this.encryptor = encryptor;
+ this.flowConfigDocBuilder = docBuilder;
+ }
+
/**
* Creates a fingerprint of a flow. The order of elements or attributes in the flow does not influence the fingerprint generation.
* This method does not accept a FlowController, which means that Processors cannot be created in order to verify default property
http://git-wip-us.apache.org/repos/asf/nifi/blob/690130b0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd
index 08038bd..4607320 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd
@@ -315,7 +315,7 @@
<xs:simpleType name="TimePeriod">
<xs:restriction base="xs:string">
- <xs:pattern value="\d+\s*(ns|nano|nanos|nanoseconds|ms|milli|millis|milliseconds|s|sec|secs|seconds|m|min|mins|minutes|h|hr|hrs|hours|d|day|days)"></xs:pattern>
+ <xs:pattern value="\d+\s*(ns|nano|nanos|nanosecond|nanoseconds|ms|milli|millis|millisecond|milliseconds|s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours|d|day|days|w|wk|wks|week|weeks)"></xs:pattern>
</xs:restriction>
</xs:simpleType>
http://git-wip-us.apache.org/repos/asf/nifi/blob/690130b0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java
index bb12e78..5b4e841 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java
@@ -16,8 +16,9 @@
*/
package org.apache.nifi.fingerprint;
+import static org.apache.nifi.fingerprint.FingerprintFactory.FLOW_CONFIG_XSD;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -26,6 +27,15 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.Before;
import org.junit.Test;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
/**
*/
@@ -42,8 +52,6 @@ public class FingerprintFactoryTest {
public void testSameFingerprint() throws IOException {
final String fp1 = fingerprinter.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
final String fp2 = fingerprinter.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1b.xml"), null);
- System.out.println(fp1);
- System.out.println(fp2);
assertEquals(fp1, fp2);
}
@@ -51,7 +59,7 @@ public class FingerprintFactoryTest {
public void testDifferentFingerprint() throws IOException {
final String fp1 = fingerprinter.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
final String fp2 = fingerprinter.createFingerprint(getResourceBytes("/nifi/fingerprint/flow2.xml"), null);
- assertFalse(fp1.equals(fp2));
+ assertNotEquals(fp1, fp2);
}
@Test
@@ -61,8 +69,48 @@ public class FingerprintFactoryTest {
assertTrue(fingerprint.contains("In Connection"));
}
+ @Test
+ public void testSchemaValidation() throws IOException {
+ FingerprintFactory fp = new FingerprintFactory(null, getValidatingDocumentBuilder());
+ final String fingerprint = fp.createFingerprint(getResourceBytes("/nifi/fingerprint/validating-flow.xml"), null);
+ }
+
private byte[] getResourceBytes(final String resource) throws IOException {
return IOUtils.toByteArray(FingerprintFactoryTest.class.getResourceAsStream(resource));
}
+ private DocumentBuilder getValidatingDocumentBuilder() {
+ final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+ documentBuilderFactory.setNamespaceAware(true);
+ final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ final Schema schema;
+ try {
+ schema = schemaFactory.newSchema(FingerprintFactory.class.getResource(FLOW_CONFIG_XSD));
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to parse schema for file flow configuration.", e);
+ }
+ try {
+ documentBuilderFactory.setSchema(schema);
+ DocumentBuilder docBuilder = documentBuilderFactory.newDocumentBuilder();
+ docBuilder.setErrorHandler(new ErrorHandler() {
+ @Override
+ public void warning(SAXParseException e) throws SAXException {
+ throw e;
+ }
+
+ @Override
+ public void error(SAXParseException e) throws SAXException {
+ throw e;
+ }
+
+ @Override
+ public void fatalError(SAXParseException e) throws SAXException {
+ throw e;
+ }
+ });
+ return docBuilder;
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to create document builder for flow configuration.", e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/690130b0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/validating-flow.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/validating-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/validating-flow.xml
new file mode 100644
index 0000000..a762dc9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/validating-flow.xml
@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<flowController encoding-version="1.0">
+ <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
+ <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
+ <rootGroup>
+ <id>289ff050-015a-1000-b21b-a39f56e56729</id>
+ <name>NiFi Flow</name>
+ <position x="0.0" y="0.0"/>
+ <comment/>
+ <processor>
+ <id>28a67671-015a-1000-77dd-8130e16012ec</id>
+ <name>PutFile</name>
+ <position x="420.0" y="288.0"/>
+ <styles>
+ <style name="background-color">#3046f0</style>
+ </styles>
+ <comment>write a file</comment>
+ <class>org.apache.nifi.processors.standard.PutFile</class>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>3000 millisecond</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Directory</name>
+ </property>
+ <property>
+ <name>Conflict Resolution Strategy</name>
+ <value>fail</value>
+ </property>
+ <property>
+ <name>Create Missing Directories</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Maximum File Count</name>
+ </property>
+ <property>
+ <name>Last Modified Time</name>
+ </property>
+ <property>
+ <name>Permissions</name>
+ </property>
+ <property>
+ <name>Owner</name>
+ </property>
+ <property>
+ <name>Group</name>
+ </property>
+ </processor>
+ <processor>
+ <id>28a6579e-015a-1000-2856-975c9a5af27e</id>
+ <name>GetFile</name>
+ <position x="422.0" y="50.0"/>
+ <styles>
+ <style name="background-color">#eb7575</style>
+ </styles>
+ <comment/>
+ <class>org.apache.nifi.processors.standard.GetFile</class>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1000000000 nanosecond</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Input Directory</name>
+ </property>
+ <property>
+ <name>File Filter</name>
+ <value>[^\.].*</value>
+ </property>
+ <property>
+ <name>Path Filter</name>
+ </property>
+ <property>
+ <name>Batch Size</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>Keep Source File</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Recurse Subdirectories</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Polling Interval</name>
+ <value>0 sec</value>
+ </property>
+ <property>
+ <name>Ignore Hidden Files</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Minimum File Age</name>
+ <value>0 sec</value>
+ </property>
+ <property>
+ <name>Maximum File Age</name>
+ </property>
+ <property>
+ <name>Minimum File Size</name>
+ <value>0 B</value>
+ </property>
+ <property>
+ <name>Maximum File Size</name>
+ </property>
+ </processor>
+ <connection>
+ <id>28a77d72-015a-1000-32e4-c6fe548f5c9d</id>
+ <name/>
+ <bendPoints>
+ <bendPoint x="815.0" y="231.0"/>
+ </bendPoints>
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>28a6579e-015a-1000-2856-975c9a5af27e</sourceId>
+ <sourceGroupId>289ff050-015a-1000-b21b-a39f56e56729</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>28a67671-015a-1000-77dd-8130e16012ec</destinationId>
+ <destinationGroupId>289ff050-015a-1000-b21b-a39f56e56729</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>1 hour</flowFileExpiration>
+ <queuePrioritizerClass>org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer</queuePrioritizerClass>
+ </connection>
+ </rootGroup>
+ <controllerServices/>
+ <reportingTasks/>
+</flowController>