You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2015/03/06 05:21:15 UTC
[01/50] incubator-nifi git commit: Adjusting error where the return
type of string was omitted from the possible selections
Repository: incubator-nifi
Updated Branches:
refs/heads/NIFI-353 48192289e -> 347e4e024
Adjusting error where the return type of string was omitted from the possible selections
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/5b145e10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/5b145e10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/5b145e10
Branch: refs/heads/NIFI-353
Commit: 5b145e10e8508bec5293aab7318978fc3c9e2b57
Parents: 7e58130
Author: Aldrin Piri <al...@gmail.com>
Authored: Mon Feb 16 18:34:23 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Mon Feb 16 18:34:23 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/processors/standard/EvaluateJsonPath.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5b145e10/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 13eb50b..d6bcdc1 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -72,7 +72,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
.name("Return Type")
.description("Indicates the desired return type of the JSON Path expressions. Selecting 'auto-detect' will set the return type to 'json' for a Destination of 'flowfile-content', and 'string' for a Destination of 'flowfile-attribute'.")
.required(true)
- .allowableValues(RETURN_TYPE_AUTO, RETURN_TYPE_AUTO, RETURN_TYPE_STRING)
+ .allowableValues(RETURN_TYPE_AUTO, RETURN_TYPE_JSON, RETURN_TYPE_STRING)
.defaultValue(RETURN_TYPE_AUTO)
.build();
[23/50] incubator-nifi git commit: NIFI-221: Fixed bugs and pulled in
changes from develop
Posted by mc...@apache.org.
NIFI-221: Fixed bugs and pulled in changes from develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/2d483238
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/2d483238
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/2d483238
Branch: refs/heads/NIFI-353
Commit: 2d48323878436c500fc90fd1d839740248d0f63d
Parents: 739f0c2
Author: Mark Payne <ma...@hotmail.com>
Authored: Thu Feb 26 09:05:28 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Thu Feb 26 09:05:28 2015 -0500
----------------------------------------------------------------------
nifi/nifi-assembly/pom.xml | 5 +
.../nifi-standard-processors/pom.xml | 4 +
.../processors/standard/HandleHttpRequest.java | 519 +++++++++++++++++++
.../processors/standard/HandleHttpResponse.java | 178 +++++++
.../org.apache.nifi.processor.Processor | 2 +
.../nifi-http-context-map-api/pom.xml | 23 +
.../org/apache/nifi/http/HttpContextMap.java | 72 +++
.../nifi-http-context-map-nar/pom.xml | 37 ++
.../nifi-http-context-map/pom.xml | 44 ++
.../nifi/http/StandardHttpContextMap.java | 176 +++++++
...org.apache.nifi.controller.ControllerService | 1 +
.../index.html | 67 +++
.../nifi-http-context-map-bundle/pom.xml | 31 ++
.../nifi-standard-services-api-nar/pom.xml | 5 +
.../nifi-standard-services/pom.xml | 2 +
nifi/nifi-nar-bundles/pom.xml | 11 +
nifi/pom.xml | 6 +
17 files changed, 1183 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-assembly/pom.xml b/nifi/nifi-assembly/pom.xml
index 9a55cbc..d65f294 100644
--- a/nifi/nifi-assembly/pom.xml
+++ b/nifi/nifi-assembly/pom.xml
@@ -156,6 +156,11 @@
<artifactId>nifi-kafka-nar</artifactId>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-nar</artifactId>
+ <type>nar</type>
+ </dependency>
</dependencies>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index a482e19..0ccb3dd 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -48,6 +48,10 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-distributed-cache-client-service-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-api</artifactId>
+ </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
new file mode 100644
index 0000000..85d7433
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -0,0 +1,519 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.Principal;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import javax.security.cert.X509Certificate;
+import javax.servlet.AsyncContext;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.http.HttpContextMap;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.ssl.SSLContextService;
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+import com.sun.jersey.api.client.ClientResponse.Status;
+
+@Tags({"http", "https", "request", "listen", "ingress", "web service"})
+@CapabilityDescription("Starts an HTTP Server and listens for HTTP Requests. For each request, creates a FlowFile and transfers to 'success'. This Processor is designed to be used in conjunction with the HandleHttpResponse Processor in order to create a Web Service")
+public class HandleHttpRequest extends AbstractProcessor {
+ public static final String HTTP_CONTEXT_ID = "http.context.identifier";
+
+ // Allowable values for client auth
+ public static final AllowableValue CLIENT_NONE = new AllowableValue("No Authentication", "Processor will not authenticate clients. Anyone can communicate with this Processor anonymously");
+ public static final AllowableValue CLIENT_WANT = new AllowableValue("Want Authentication", "Processor will try to verify the client but if unable to verify will allow the client to communicate anonymously");
+ public static final AllowableValue CLIENT_NEED = new AllowableValue("Need Authentication", "Processor will reject communications from any client unless the client provides a certificate that is trusted by the TrustStore specified in the SSL Context Service");
+
+
+ public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder()
+ .name("Listening Port")
+ .description("The Port to listen on for incoming HTTP requests")
+ .required(true)
+ .addValidator(StandardValidators.createLongValidator(0L, 65535L, true))
+ .expressionLanguageSupported(false)
+ .defaultValue("80")
+ .build();
+ public static final PropertyDescriptor HOSTNAME = new PropertyDescriptor.Builder()
+ .name("Hostname")
+ .description("The Hostname to bind to. If not specified, will bind to all hosts")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(false)
+ .build();
+ public static final PropertyDescriptor HTTP_CONTEXT_MAP = new PropertyDescriptor.Builder()
+ .name("HTTP Context Map")
+ .description("The HTTP Context Map Controller Service to use for caching the HTTP Request Information")
+ .required(true)
+ .identifiesControllerService(HttpContextMap.class)
+ .build();
+ public static final PropertyDescriptor SSL_CONTEXT = new PropertyDescriptor.Builder()
+ .name("SSL Context Service")
+ .description("The SSL Context Service to use in order to secure the server. If specified, the server will accept only HTTPS requests; otherwise, the server will accept only HTTP requests")
+ .required(false)
+ .identifiesControllerService(SSLContextService.class)
+ .build();
+ public static final PropertyDescriptor PATH_REGEX = new PropertyDescriptor.Builder()
+ .name("Allowed Paths")
+ .description("A Regular Expression that specifies the valid HTTP Paths that are allowed in the incoming URL Requests. If this value is specified and the path of the HTTP Requests does not match this Regular Expression, the Processor will respond with a 404: NotFound")
+ .required(false)
+ .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .expressionLanguageSupported(false)
+ .build();
+ public static final PropertyDescriptor ALLOW_GET = new PropertyDescriptor.Builder()
+ .name("Allow GET")
+ .description("Allow HTTP GET Method")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .build();
+ public static final PropertyDescriptor ALLOW_POST = new PropertyDescriptor.Builder()
+ .name("Allow POST")
+ .description("Allow HTTP POST Method")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .build();
+ public static final PropertyDescriptor ALLOW_PUT = new PropertyDescriptor.Builder()
+ .name("Allow PUT")
+ .description("Allow HTTP PUT Method")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .build();
+ public static final PropertyDescriptor ALLOW_DELETE = new PropertyDescriptor.Builder()
+ .name("Allow DELETE")
+ .description("Allow HTTP DELETE Method")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .build();
+ public static final PropertyDescriptor ALLOW_HEAD = new PropertyDescriptor.Builder()
+ .name("Allow HEAD")
+ .description("Allow HTTP HEAD Method")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("false")
+ .build();
+ public static final PropertyDescriptor ALLOW_OPTIONS = new PropertyDescriptor.Builder()
+ .name("Allow OPTIONS")
+ .description("Allow HTTP OPTIONS Method")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("false")
+ .build();
+ public static final PropertyDescriptor ADDITIONAL_METHODS = new PropertyDescriptor.Builder()
+ .name("Additional HTTP Methods")
+ .description("A comma-separated list of non-standard HTTP Methods that should be allowed")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(false)
+ .build();
+ public static final PropertyDescriptor CLIENT_AUTH = new PropertyDescriptor.Builder()
+ .name("Client Authentication")
+ .description("Specifies whether or not the Processor should authenticate clients. This value is ignored if the <SSL Context Service> Property is not specified or the SSL Context provided uses only a KeyStore and not a TrustStore.")
+ .required(true)
+ .allowableValues(CLIENT_NONE, CLIENT_WANT, CLIENT_NEED)
+ .defaultValue(CLIENT_NONE.getValue())
+ .build();
+
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("All content that is received is routed to the 'success' relationship")
+ .build();
+
+ private volatile Server server;
+ private final BlockingQueue<HttpRequestContainer> containerQueue = new LinkedBlockingQueue<>(50);
+
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> descriptors = new ArrayList<>();
+ descriptors.add(PORT);
+ descriptors.add(HOSTNAME);
+ descriptors.add(SSL_CONTEXT);
+ descriptors.add(HTTP_CONTEXT_MAP);
+ descriptors.add(PATH_REGEX);
+ descriptors.add(ALLOW_GET);
+ descriptors.add(ALLOW_POST);
+ descriptors.add(ALLOW_PUT);
+ descriptors.add(ALLOW_DELETE);
+ descriptors.add(ALLOW_HEAD);
+ descriptors.add(ALLOW_OPTIONS);
+ descriptors.add(ADDITIONAL_METHODS);
+
+ return descriptors;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return Collections.singleton(REL_SUCCESS);
+ }
+
+
+ @OnScheduled
+ public void initializeServer(final ProcessContext context) throws Exception {
+ final String host = context.getProperty(HOSTNAME).getValue();
+ final int port = context.getProperty(PORT).asInteger();
+ final SSLContextService sslService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);
+
+ final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
+ final boolean need;
+ final boolean want;
+ if ( CLIENT_NEED.equals(clientAuthValue) ) {
+ need = true;
+ want = false;
+ } else if ( CLIENT_WANT.equals(clientAuthValue) ) {
+ need = false;
+ want = true;
+ } else {
+ need = false;
+ want = false;
+ }
+
+ final SslContextFactory sslFactory = (sslService == null) ? null : createSslFactory(sslService, need, want);
+ final Server server = new Server(port);
+
+ // create the http configuration
+ final HttpConfiguration httpConfiguration = new HttpConfiguration();
+ if ( sslFactory == null ) {
+ // create the connector
+ final ServerConnector http = new ServerConnector(server, new HttpConnectionFactory(httpConfiguration));
+
+ // set host and port
+ if (StringUtils.isNotBlank(host)) {
+ http.setHost(host);
+ }
+ http.setPort(port);
+
+ // add this connector
+ server.setConnectors(new Connector[] {http});
+ } else {
+ // add some secure config
+ final HttpConfiguration httpsConfiguration = new HttpConfiguration(httpConfiguration);
+ httpsConfiguration.setSecureScheme("https");
+ httpsConfiguration.setSecurePort(port);
+ httpsConfiguration.addCustomizer(new SecureRequestCustomizer());
+
+ // build the connector
+ final ServerConnector https = new ServerConnector(server,
+ new SslConnectionFactory(sslFactory, "http/1.1"),
+ new HttpConnectionFactory(httpsConfiguration));
+
+ // set host and port
+ if (StringUtils.isNotBlank(host)) {
+ https.setHost(host);
+ }
+ https.setPort(port);
+
+ // add this connector
+ server.setConnectors(new Connector[] {https});
+ }
+
+ final Set<String> allowedMethods = new HashSet<>();
+ if ( context.getProperty(ALLOW_GET).asBoolean() ) {
+ allowedMethods.add("GET");
+ }
+ if ( context.getProperty(ALLOW_POST).asBoolean() ) {
+ allowedMethods.add("POST");
+ }
+ if ( context.getProperty(ALLOW_PUT).asBoolean() ) {
+ allowedMethods.add("PUT");
+ }
+ if ( context.getProperty(ALLOW_DELETE).asBoolean() ) {
+ allowedMethods.add("DELETE");
+ }
+ if ( context.getProperty(ALLOW_HEAD).asBoolean() ) {
+ allowedMethods.add("HEAD");
+ }
+ if ( context.getProperty(ALLOW_OPTIONS).asBoolean() ) {
+ allowedMethods.add("OPTIONS");
+ }
+
+ final String additionalMethods = context.getProperty(ADDITIONAL_METHODS).getValue();
+ if ( additionalMethods != null ) {
+ for ( final String additionalMethod : additionalMethods.split(",") ) {
+ final String trimmed = additionalMethod.trim();
+ if ( !trimmed.isEmpty() ) {
+ allowedMethods.add(trimmed.toUpperCase());
+ }
+ }
+ }
+
+ final String pathRegex = context.getProperty(PATH_REGEX).getValue();
+ final Pattern pathPattern = (pathRegex == null) ? null : Pattern.compile(pathRegex);
+
+ server.setHandler(new AbstractHandler() {
+ @Override
+ public void handle(final String target, final Request baseRequest, final HttpServletRequest request,
+ final HttpServletResponse response) throws IOException, ServletException {
+
+ final String requestUri = request.getRequestURI();
+ if ( !allowedMethods.contains(request.getMethod().toUpperCase()) ) {
+ getLogger().info("Sending back METHOD_NOT_ALLOWED response to {}; method was {}; request URI was {}",
+ new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
+ response.sendError(Status.METHOD_NOT_ALLOWED.getStatusCode());
+ return;
+ }
+
+ if ( pathPattern != null ) {
+ final URI uri;
+ try {
+ uri = new URI(requestUri);
+ } catch (final URISyntaxException e) {
+ throw new ServletException(e);
+ }
+
+ if ( !pathPattern.matcher(uri.getPath()).matches() ) {
+ response.sendError(Status.NOT_FOUND.getStatusCode());
+ getLogger().info("Sending back NOT_FOUND response to {}; request was {} {}",
+ new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
+ return;
+ }
+ }
+
+ // If destination queues full, send back a 503: Service Unavailable.
+ if ( context.getAvailableRelationships().isEmpty() ) {
+ response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ return;
+ }
+
+ // Right now, that information, though, is only in the ProcessSession, not the ProcessContext,
+ // so it is not known to us. Should see if it can be added to the ProcessContext.
+ final AsyncContext async = baseRequest.startAsync();
+ final boolean added = containerQueue.offer(new HttpRequestContainer(request, response, async));
+
+ if ( added ) {
+ getLogger().debug("Added Http Request to queue for {} {} from {}", new Object[] {request.getMethod(), requestUri, request.getRemoteAddr()});
+ } else {
+ getLogger().info("Sending back a SERVICE_UNAVAILABLE response to {}; request was {} {}",
+ new Object[] {request.getRemoteAddr(), request.getMethod(), request.getRemoteAddr()});
+
+ response.sendError(Status.SERVICE_UNAVAILABLE.getStatusCode());
+ response.flushBuffer();
+ async.complete();
+ }
+ }
+ });
+
+ this.server = server;
+ server.start();
+
+ getLogger().info("Server started and listening on port " + getPort());
+ }
+
+ protected int getPort() {
+ for ( final Connector connector : server.getConnectors() ) {
+ if ( connector instanceof ServerConnector ) {
+ return ((ServerConnector) connector).getPort();
+ }
+ }
+
+ throw new IllegalStateException("Server is not listening on any ports");
+ }
+
+ protected int getRequestQueueSize() {
+ return containerQueue.size();
+ }
+
+ private SslContextFactory createSslFactory(final SSLContextService sslService, final boolean needClientAuth, final boolean wantClientAuth) {
+ final SslContextFactory sslFactory = new SslContextFactory();
+
+ sslFactory.setNeedClientAuth(needClientAuth);
+ sslFactory.setWantClientAuth(wantClientAuth);
+
+ if ( sslService.isKeyStoreConfigured() ) {
+ sslFactory.setKeyStorePath(sslService.getKeyStoreFile());
+ sslFactory.setKeyStorePassword(sslService.getKeyStorePassword());
+ sslFactory.setKeyStoreType(sslService.getKeyStoreType());
+ }
+
+ if ( sslService.isTrustStoreConfigured() ) {
+ sslFactory.setTrustStorePath(sslService.getTrustStoreFile());
+ sslFactory.setTrustStorePassword(sslService.getTrustStorePassword());
+ sslFactory.setTrustStoreType(sslService.getTrustStoreType());
+ }
+
+ return sslFactory;
+ }
+
+ @OnStopped
+ public void shutdown() throws Exception {
+ if ( server != null ) {
+ getLogger().debug("Shutting down server");
+ server.stop();
+ server.join();
+ getLogger().info("Shut down {}", new Object[] {server});
+ }
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ final HttpRequestContainer container = containerQueue.poll();
+ if ( container == null ) {
+ return;
+ }
+
+ final long start = System.nanoTime();
+ final HttpServletRequest request = container.getRequest();
+ FlowFile flowFile = session.create();
+ try {
+ flowFile = session.importFrom(request.getInputStream(), flowFile);
+ } catch (final IOException e) {
+ getLogger().error("Failed to receive content from HTTP Request from {} due to {}", new Object[] {request.getRemoteAddr(), e});
+ session.remove(flowFile);
+ return;
+ }
+
+ final String contextIdentifier = UUID.randomUUID().toString();
+ final Map<String, String> attributes = new HashMap<>();
+ putAttribute(attributes, HTTP_CONTEXT_ID, contextIdentifier);
+ putAttribute(attributes, "mime.type", request.getContentType());
+ putAttribute(attributes, "http.servlet.path", request.getServletPath());
+ putAttribute(attributes, "http.context.path", request.getContextPath());
+ putAttribute(attributes, "http.method", request.getMethod());
+ putAttribute(attributes, "http.query.string", request.getQueryString());
+ putAttribute(attributes, "http.remote.host", request.getRemoteHost());
+ putAttribute(attributes, "http.remote.addr", request.getRemoteAddr());
+ putAttribute(attributes, "http.remote.user", request.getRemoteUser());
+ putAttribute(attributes, "http.request.uri", request.getRequestURI());
+ putAttribute(attributes, "http.auth.type", request.getAuthType());
+
+ final Enumeration<String> headerNames = request.getHeaderNames();
+ while ( headerNames.hasMoreElements() ) {
+ final String headerName = headerNames.nextElement();
+ final String headerValue = request.getHeader(headerName);
+ putAttribute(attributes, "http.headers." + headerName, headerValue);
+ }
+
+ final Principal principal = request.getUserPrincipal();
+ if ( principal != null ) {
+ putAttribute(attributes, "http.principal.name", principal.getName());
+ }
+
+ final X509Certificate certs[] = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
+ final String subjectDn;
+ if ( certs != null && certs.length > 0 ) {
+ final X509Certificate cert = certs[0];
+ subjectDn = cert.getSubjectDN().getName();
+ final String issuerDn = cert.getIssuerDN().getName();
+
+ putAttribute(attributes, "http.subject.dn", subjectDn);
+ putAttribute(attributes, "http.issuer.dn", issuerDn);
+ } else {
+ subjectDn = null;
+ }
+
+ flowFile = session.putAllAttributes(flowFile, attributes);
+
+ final HttpContextMap contextMap = context.getProperty(HTTP_CONTEXT_MAP).asControllerService(HttpContextMap.class);
+ final boolean registered = contextMap.register(contextIdentifier, request, container.getResponse(), container.getContext());
+
+ if ( !registered ) {
+ getLogger().warn("Received request from {} but could not process it because too many requests are already outstanding; responding with SERVICE_UNAVAILABLE", new Object[] {request.getRemoteAddr()});
+
+ try {
+ container.getResponse().setStatus(Status.SERVICE_UNAVAILABLE.getStatusCode());
+ container.getResponse().flushBuffer();
+ container.getContext().complete();
+ } catch (final Exception e) {
+ getLogger().warn("Failed to respond with SERVICE_UNAVAILABLE message to {} due to {}", new Object[] {request.getRemoteAddr(), e});
+ }
+
+ return;
+ }
+
+ final long receiveMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+ session.getProvenanceReporter().receive(flowFile, request.getRequestURI(), "Received from " + request.getRemoteAddr() + (subjectDn == null ? "" : " with DN=" + subjectDn), receiveMillis);
+ session.transfer(flowFile, REL_SUCCESS);
+ getLogger().info("Transferring {} to 'success'; received from {}", new Object[] {flowFile, request.getRemoteAddr()});
+ }
+
+ private void putAttribute(final Map<String, String> map, final String key, final String value) {
+ if ( value == null ) {
+ return;
+ }
+
+ map.put(key, value);
+ }
+
+
+ private static class HttpRequestContainer {
+ private final HttpServletRequest request;
+ private final HttpServletResponse response;
+ private final AsyncContext context;
+
+ public HttpRequestContainer(final HttpServletRequest request, final HttpServletResponse response, final AsyncContext async) {
+ this.request = request;
+ this.response = response;
+ this.context = async;
+ }
+
+ public HttpServletRequest getRequest() {
+ return request;
+ }
+
+ public HttpServletResponse getResponse() {
+ return response;
+ }
+
+ public AsyncContext getContext() {
+ return context;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpResponse.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpResponse.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpResponse.java
new file mode 100644
index 0000000..fd55927
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpResponse.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.http.HttpContextMap;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+@Tags({"http", "https", "response", "egress", "web service"})
+@CapabilityDescription("Sends an HTTP Response to the Requestor that generated a FlowFile. This Processor is designed to be used in conjunction with the HandleHttpRequest in order to create a web service.")
+public class HandleHttpResponse extends AbstractProcessor {
+ public static final Pattern NUMBER_PATTERN = Pattern.compile("[0-9]+");
+ public static final String HTTP_CONTEXT_ID = "http.context.identifier";
+
+ public static final PropertyDescriptor STATUS_CODE = new PropertyDescriptor.Builder()
+ .name("HTTP Status Code")
+ .description("The HTTP Status Code to use when responding to the HTTP Request. See Section 10 of RFC 2616 for more information.")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .build();
+ public static final PropertyDescriptor HTTP_CONTEXT_MAP = new PropertyDescriptor.Builder()
+ .name("HTTP Context Map")
+ .description("The HTTP Context Map Controller Service to use for caching the HTTP Request Information")
+ .required(true)
+ .identifiesControllerService(HttpContextMap.class)
+ .build();
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("FlowFiles will be routed to this Relationship after the response has been successfully sent to the requestor")
+ .build();
+ public static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("FlowFiles will be routed to this Relationship if the Processor is unable to respond to the requestor. This may happen, for instance, if the connection times out or if NiFi is restarted before responding to the HTTP Request.")
+ .build();
+
+
+ @Override
+ public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> properties = new ArrayList<>();
+ properties.add(STATUS_CODE);
+ properties.add(HTTP_CONTEXT_MAP);
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ final Set<Relationship> relationships = new HashSet<>();
+ relationships.add(REL_SUCCESS);
+ relationships.add(REL_FAILURE);
+ return relationships;
+ }
+
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .description("Specifies the value to send for the '" + propertyDescriptorName + "' HTTP Header")
+ .name(propertyDescriptorName)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dynamic(true)
+ .expressionLanguageSupported(true)
+ .build();
+ }
+
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+ if ( flowFile == null ) {
+ return;
+ }
+
+ final String contextIdentifier = flowFile.getAttribute(HTTP_CONTEXT_ID);
+ if ( contextIdentifier == null ) {
+ session.transfer(flowFile, REL_FAILURE);
+ getLogger().warn("Failed to respond to HTTP request for {} because FlowFile did not have an "
+ + "'http.context.identifier' attribute", new Object[] {flowFile});
+ return;
+ }
+
+ final String statusCodeValue = context.getProperty(STATUS_CODE).evaluateAttributeExpressions(flowFile).getValue();
+ if ( !isNumber(statusCodeValue) ) {
+ session.transfer(flowFile, REL_FAILURE);
+ getLogger().error("Failed to response to HTTP request for {} because status code was '{}', which is not a valid number", new Object[] {flowFile, statusCodeValue});
+ }
+
+ final HttpContextMap contextMap = context.getProperty(HTTP_CONTEXT_MAP).asControllerService(HttpContextMap.class);
+ final HttpServletResponse response = contextMap.getResponse(contextIdentifier);
+ if ( response == null ) {
+ session.transfer(flowFile, REL_FAILURE);
+ getLogger().error("Failed to respond to HTTP request for {} because FlowFile had an '{}' "
+ + "attribute of {} but could not find an HTTP Response Object for this identifier", new Object[] {flowFile, HTTP_CONTEXT_ID, contextIdentifier});
+ return;
+ }
+
+ final int statusCode = Integer.parseInt(statusCodeValue);
+ response.setStatus(statusCode);
+
+ for ( final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet() ) {
+ final PropertyDescriptor descriptor = entry.getKey();
+ if ( descriptor.isDynamic() ) {
+ final String headerName = descriptor.getName();
+ final String headerValue = context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue();
+
+ if ( !headerValue.trim().isEmpty() ) {
+ response.setHeader(headerName, headerValue);
+ }
+ }
+ }
+
+ try {
+ session.exportTo(flowFile, response.getOutputStream());
+ response.flushBuffer();
+ } catch (final IOException ioe) {
+ session.transfer(flowFile, REL_FAILURE);
+ getLogger().error("Failed to respond to HTTP request for {} due to {}", new Object[] {flowFile, ioe});
+ return;
+ }
+
+ try {
+ contextMap.complete(contextIdentifier);
+ } catch (final IllegalStateException ise) {
+ getLogger().error("Failed to complete HTTP Transaction for {} due to {}", new Object[] {flowFile, ise});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+ session.transfer(flowFile, REL_SUCCESS);
+ getLogger().info("Successfully responded to HTTP Request for {} with status code {}", new Object[] {flowFile, statusCode});
+ }
+
+ private static boolean isNumber(final String value) {
+ if ( value.length() == 0 ) {
+ return false;
+ }
+
+ for (int i=0; i < value.length(); i++) {
+ if ( !Character.isDigit(value.charAt(i)) ) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index 66c9697..9702b56 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -28,6 +28,8 @@ org.apache.nifi.processors.standard.GetFile
org.apache.nifi.processors.standard.GetFTP
org.apache.nifi.processors.standard.GetHTTP
org.apache.nifi.processors.standard.GetSFTP
+org.apache.nifi.processors.standard.HandleHttpRequest
+org.apache.nifi.processors.standard.HandleHttpResponse
org.apache.nifi.processors.standard.HashAttribute
org.apache.nifi.processors.standard.HashContent
org.apache.nifi.processors.standard.IdentifyMimeType
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/pom.xml
new file mode 100644
index 0000000..262a41b
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/pom.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-http-context-map-api</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
new file mode 100644
index 0000000..04ff6ce
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-api/src/main/java/org/apache/nifi/http/HttpContextMap.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.http;
+
+import javax.servlet.AsyncContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.nifi.controller.ControllerService;
+
+
+/**
+ * <p>
+ * An interface that provides the capability of receiving an HTTP servlet request in one component
+ * and responding to that request in another component.
+ * </p>
+ *
+ * <p>
+ * The intended flow is for the component receiving the HTTP request to register the request, response,
+ * and AsyncContext with a particular identifier via the
+ * {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)}
+ * method. Another component is then able to obtain the response
+ * by providing that identifier to the {@link #getResponse(String)} method. After writing to the
+ * HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
+ * </p>
+ */
+public interface HttpContextMap extends ControllerService {
+
+ /**
+ * Registers an HttpServletRequest, HttpServletResponse, and the AsyncContext for a given identifier
+ *
+ * @param identifier
+ * @param request
+ * @param response
+ * @param context
+ *
+ * @return true if register is successful, false if the context map is too full because too many requests have already been received and not processed
+ *
+ * @throws IllegalStateException if the identifier is already registered
+ */
+ boolean register(String identifier, HttpServletRequest request, HttpServletResponse response, AsyncContext context);
+
+ /**
+ * Retrieves the HttpServletResponse for the given identifier, if it exists
+ * @param identifier
+ * @return the HttpServletResponse for the given identifier, or {@code null} if it does not exist
+ */
+ HttpServletResponse getResponse(String identifier);
+
+ /**
+ * Marks the HTTP request/response for the given identifier as complete
+ * @param identifier
+ *
+ * @throws IllegalStateException if the identifier is not registered to a valid AsyncContext
+ */
+ void complete(String identifier);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map-nar/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map-nar/pom.xml
new file mode 100644
index 0000000..653c218
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map-nar/pom.xml
@@ -0,0 +1,37 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-bundle</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-http-context-map-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/pom.xml
new file mode 100644
index 0000000..abf78db
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/pom.xml
@@ -0,0 +1,44 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-bundle</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-http-context-map</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-processor-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
new file mode 100644
index 0000000..c73b98b
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/java/org/apache/nifi/http/StandardHttpContextMap.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.http;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import javax.servlet.AsyncContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.nifi.annotation.lifecycle.OnDisabled;
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.processor.util.StandardValidators;
+
+public class StandardHttpContextMap extends AbstractControllerService implements HttpContextMap {
+ public static final PropertyDescriptor MAX_OUTSTANDING_REQUESTS = new PropertyDescriptor.Builder()
+ .name("Maximum Outstanding Requests")
+ .description("The maximum number of HTTP requests that can be outstanding at any one time. Any attempt to register an additional HTTP Request will cause an error")
+ .required(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .defaultValue("5000")
+ .build();
+ public static final PropertyDescriptor REQUEST_EXPIRATION = new PropertyDescriptor.Builder()
+ .name("Request Expiration")
+ .description("Specifies how long an HTTP Request should be left unanswered before being evicted from the cache and being responded to with a Service Unavailable status code")
+ .required(true)
+ .expressionLanguageSupported(false)
+ .defaultValue("1 min")
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .build();
+
+ private final ConcurrentMap<String, Wrapper> wrapperMap = new ConcurrentHashMap<>();
+
+ private volatile int maxSize = 5000;
+ private volatile long maxRequestNanos;
+ private volatile ScheduledExecutorService executor;
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> properties = new ArrayList<>(2);
+ properties.add(MAX_OUTSTANDING_REQUESTS);
+ properties.add(REQUEST_EXPIRATION);
+ return properties;
+ }
+
+ @OnEnabled
+ public void onConfigured(final ConfigurationContext context) {
+ maxSize = context.getProperty(MAX_OUTSTANDING_REQUESTS).asInteger();
+ executor = Executors.newSingleThreadScheduledExecutor();
+
+ maxRequestNanos = context.getProperty(REQUEST_EXPIRATION).asTimePeriod(TimeUnit.NANOSECONDS);
+ final long scheduleNanos = maxRequestNanos / 2;
+ executor.scheduleWithFixedDelay(new CleanupExpiredRequests(), scheduleNanos, scheduleNanos, TimeUnit.NANOSECONDS);
+ }
+
+ @OnDisabled
+ public void cleanup() {
+ if ( executor != null ) {
+ executor.shutdown();
+ }
+ }
+
+ @Override
+ public boolean register(final String identifier, final HttpServletRequest request, final HttpServletResponse response, final AsyncContext context) {
+ // fail if there are too many already. Maybe add a configuration property for how many
+ // outstanding, with a default of say 5000
+ if ( wrapperMap.size() >= maxSize ) {
+ return false;
+ }
+ final Wrapper wrapper = new Wrapper(request, response, context);
+ final Wrapper existing = wrapperMap.putIfAbsent(identifier, wrapper);
+ if ( existing != null ) {
+ throw new IllegalStateException("HTTP Request already registered with identifier " + identifier);
+ }
+
+ return true;
+ }
+
+ @Override
+ public HttpServletResponse getResponse(final String identifier) {
+ final Wrapper wrapper = wrapperMap.get(identifier);
+ if ( wrapper == null ) {
+ return null;
+ }
+
+ return wrapper.getResponse();
+ }
+
+ @Override
+ public void complete(final String identifier) {
+ final Wrapper wrapper = wrapperMap.remove(identifier);
+ if ( wrapper == null ) {
+ throw new IllegalStateException("No HTTP Request registered with identifier " + identifier);
+ }
+
+ wrapper.getAsync().complete();
+ }
+
+ private static class Wrapper {
+ @SuppressWarnings("unused")
+ private final HttpServletRequest request;
+ private final HttpServletResponse response;
+ private final AsyncContext async;
+ private final long nanoTimeAdded = System.nanoTime();
+
+ public Wrapper(final HttpServletRequest request, final HttpServletResponse response, final AsyncContext async) {
+ this.request = request;
+ this.response = response;
+ this.async = async;
+ }
+
+ public HttpServletResponse getResponse() {
+ return response;
+ }
+
+ public AsyncContext getAsync() {
+ return async;
+ }
+
+ public long getNanoTimeAdded() {
+ return nanoTimeAdded;
+ }
+ }
+
+ private class CleanupExpiredRequests implements Runnable {
+ @Override
+ public void run() {
+ final long now = System.nanoTime();
+ final long threshold = now - maxRequestNanos;
+
+ final Iterator<Map.Entry<String, Wrapper>> itr = wrapperMap.entrySet().iterator();
+ while ( itr.hasNext() ) {
+ final Map.Entry<String, Wrapper> entry = itr.next();
+ if ( entry.getValue().getNanoTimeAdded() < threshold ) {
+ itr.remove();
+
+ // send SERVICE_UNAVAILABLE
+ try {
+ final AsyncContext async = entry.getValue().getAsync();
+ ((HttpServletResponse) async.getResponse()).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ async.complete();
+ } catch (final IOException ioe) {
+ // we are trying to indicate that we are unavailable. If we have an IOException and cannot respond,
+ // then so be it. Nothing to really do here.
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
new file mode 100644
index 0000000..8c5721c
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
@@ -0,0 +1 @@
+org.apache.nifi.http.StandardHttpContextMap
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/docs/org.apache.nifi.http.StandardHttpContextMap/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/docs/org.apache.nifi.http.StandardHttpContextMap/index.html b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/docs/org.apache.nifi.http.StandardHttpContextMap/index.html
new file mode 100644
index 0000000..5c8b83a
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/nifi-http-context-map/src/main/resources/docs/org.apache.nifi.http.StandardHttpContextMap/index.html
@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html lang="en">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <head>
+ <meta charset="utf-8" />
+ <title>StandardHttpContextMap</title>
+
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
+ </head>
+
+ <body>
+ <h2>Description:</h2>
+ <p>
+ This is the standard implementation of the SSL Context Map. This service is used to provide
+ coordination between
+ <a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a>
+ and
+ <a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a>
+ Processors.
+ </p>
+
+ <!-- Service Documentation ================================================== -->
+ <h2>Configuring the HTTP Context Map:</h2>
+ <p>
+ The <code>controller-services.xml</code> file is located in the NiFi <code>conf</code>
+ directory. The user may set up any number of controller services within this file.
+ </p>
+
+ <p>
+ This controller service exposes a single property named <code>Maximum Outstanding Requests</code>.
+ This property determines the maximum number of HTTP requests that can be outstanding at any one time.
+ Any attempt to register an additional HTTP Request will cause an error. The default value is 5000.
+ Below is an example of the template for a StandardHttpContextMap controller service.
+ </p>
+
+ <pre>
+<?xml version="1.0" encoding="UTF-8" ?>
+<services>
+ <service>
+ <identifier>http-context-map</identifier>
+ <class>org.apache.nifi.http.StandardHttpContextMap</class>
+ <property name="Maximum Outstanding Requests">5000</property>
+ </service>
+</services>
+ </pre>
+
+ <p>
+ <strong>See Also:</strong><br />
+ <a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a><br />
+ <a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a><br />
+ </p>
+
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/pom.xml
new file mode 100644
index 0000000..5539bd8
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-http-context-map-bundle/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-http-context-map-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-http-context-map</module>
+ <module>nifi-http-context-map-nar</module>
+ </modules>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml
index 0da53fc..7668cda 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml
@@ -37,5 +37,10 @@
<artifactId>nifi-load-distribution-service-api</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-api</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/nifi-standard-services/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-services/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-services/pom.xml
index 449170c..47762b3 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-services/pom.xml
+++ b/nifi/nifi-nar-bundles/nifi-standard-services/pom.xml
@@ -26,8 +26,10 @@
<module>nifi-distributed-cache-client-service-api</module>
<module>nifi-distributed-cache-services-bundle</module>
<module>nifi-load-distribution-service-api</module>
+ <module>nifi-http-context-map-api</module>
<module>nifi-ssl-context-bundle</module>
<module>nifi-ssl-context-service-api</module>
+ <module>nifi-http-context-map-bundle</module>
<module>nifi-standard-services-api-nar</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/nifi-nar-bundles/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/pom.xml b/nifi/nifi-nar-bundles/pom.xml
index 229353f..758fe58 100644
--- a/nifi/nifi-nar-bundles/pom.xml
+++ b/nifi/nifi-nar-bundles/pom.xml
@@ -60,6 +60,12 @@
<version>0.0.2-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-api</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-distributed-cache-protocol</artifactId>
@@ -75,6 +81,11 @@
<artifactId>nifi-ssl-context-service</artifactId>
<version>0.0.2-incubating-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-volatile-provenance-repository</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2d483238/nifi/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/pom.xml b/nifi/pom.xml
index 5881db7..3e42e28 100644
--- a/nifi/pom.xml
+++ b/nifi/pom.xml
@@ -753,6 +753,12 @@
<version>0.0.2-incubating-SNAPSHOT</version>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-nar</artifactId>
+ <version>0.0.2-incubating-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
[20/50] incubator-nifi git commit: Adding an abstract class to serve
as a base class for JsonPath related processors and preferring this for much
of the functionality present in JsonUtils.
Posted by mc...@apache.org.
Adding an abstract class to serve as a base class for JsonPath related processors and preferring this for much of the functionality present in JsonUtils.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/46bf048b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/46bf048b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/46bf048b
Branch: refs/heads/NIFI-353
Commit: 46bf048b2407ddbd61024d2d0e69beccb2f72014
Parents: 81234f3
Author: Aldrin Piri <al...@gmail.com>
Authored: Sat Feb 21 20:26:58 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sat Feb 21 20:26:58 2015 -0500
----------------------------------------------------------------------
.../standard/AbstractJsonPathProcessor.java | 110 +++++++++++++++++++
.../processors/standard/EvaluateJsonPath.java | 17 +--
.../nifi/processors/standard/SplitJson.java | 14 ++-
.../processors/standard/util/JsonUtils.java | 82 +-------------
4 files changed, 129 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/46bf048b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
new file mode 100644
index 0000000..cb4dcf6
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.InvalidPathException;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.spi.json.JsonProvider;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processors.standard.util.JsonUtils;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.util.BooleanHolder;
+import org.apache.nifi.util.ObjectHolder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Provides common functionality used for processors interacting and manipulating JSON data via JsonPath.
+ *
+ * @see <a href="http://json.org">http://json.org</a>
+ * @see <a href="https://github.com/jayway/JsonPath">https://github.com/jayway/JsonPath</a>
+ */
+public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
+
+ protected static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
+
+ public static final Validator JSON_PATH_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ String error = null;
+ try {
+ JsonPath compile = JsonPath.compile(input);
+ } catch (InvalidPathException ipe) {
+ error = ipe.toString();
+ }
+ return new ValidationResult.Builder().valid(error == null).explanation(error).build();
+ }
+ };
+
+ public static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
+
+ final BooleanHolder validJsonHolder = new BooleanHolder(false);
+ processSession.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(InputStream in) throws IOException {
+ validJsonHolder.set(JsonUtils.isValidJson(in));
+ }
+ });
+
+ // Parse the document once into an associated context to support multiple path evaluations if specified
+ final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
+
+ if (validJsonHolder.get()) {
+ processSession.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(InputStream in) throws IOException {
+ try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) {
+ DocumentContext ctx = JsonPath.parse(in);
+ contextHolder.set(ctx);
+ }
+ }
+ });
+ }
+
+ return contextHolder.get();
+ }
+
+ /**
+ * Determines the context by which JsonSmartJsonProvider would treat the value. {@link java.util.Map} and
+ * {@link java.util.List} objects can be rendered as JSON elements, everything else is treated as a scalar.
+ *
+ * @param obj item to be inspected if it is a scalar or a JSON element
+ * @return false, if the object is a supported type; true otherwise
+ */
+ static boolean isJsonScalar(Object obj) {
+ // For the default provider, JsonSmartJsonProvider, a Map or List is able to be handled as a JSON entity
+ return !(obj instanceof Map || obj instanceof List);
+ }
+
+ public static String getResultRepresentation(Object jsonPathResult) {
+ if (isJsonScalar(jsonPathResult)) {
+ return jsonPathResult.toString();
+ }
+ return JSON_PROVIDER.toJson(jsonPathResult);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/46bf048b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 65266ff..f7caa68 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -29,10 +29,12 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
-import org.apache.nifi.processor.*;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
-import org.apache.nifi.processors.standard.util.JsonUtils;
import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.util.ObjectHolder;
import org.apache.nifi.util.StringUtils;
@@ -57,7 +59,7 @@ import java.util.*;
+ "If Destination is 'flowfile-content' and the JsonPath does not evaluate to a defined path, the FlowFile will be routed to 'unmatched' without having its contents modified. "
+ "If Destination is flowfile-attribute and the expression matches nothing, attributes will be created with "
+ "empty strings as the value, and the FlowFile will always be routed to 'matched.'")
-public class EvaluateJsonPath extends AbstractProcessor {
+public class EvaluateJsonPath extends AbstractJsonPathProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
public static final String DESTINATION_CONTENT = "flowfile-content";
@@ -142,7 +144,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.expressionLanguageSupported(false)
- .addValidator(JsonUtils.JSON_PATH_VALIDATOR)
+ .addValidator(JSON_PATH_VALIDATOR)
.required(false)
.dynamic(true)
.build();
@@ -175,7 +177,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
returnType = destination.equals(DESTINATION_CONTENT) ? RETURN_TYPE_JSON : RETURN_TYPE_SCALAR;
}
- final DocumentContext documentContext = JsonUtils.validateAndEstablishJsonContext(processSession, flowFile);
+ final DocumentContext documentContext = validateAndEstablishJsonContext(processSession, flowFile);
if (documentContext == null) {
logger.error("FlowFile {} did not have valid JSON content.", new Object[]{flowFile});
@@ -194,7 +196,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
try {
Object result = documentContext.read(jsonPathExp);
- if (returnType.equals(RETURN_TYPE_SCALAR) && !JsonUtils.isJsonScalar(result)) {
+ if (returnType.equals(RETURN_TYPE_SCALAR) && !isJsonScalar(result)) {
logger.error("Unable to return a scalar value for the expression {} for FlowFile {}. Evaluated value was {}. Transferring to {}.",
new Object[]{jsonPathExp.getPath(), flowFile.getId(), result.toString(), REL_FAILURE.getName()});
processSession.transfer(flowFile, REL_FAILURE);
@@ -212,7 +214,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
}
- final String resultRepresentation = JsonUtils.getResultRepresentation(resultHolder.get());
+ final String resultRepresentation = getResultRepresentation(resultHolder.get());
switch (destination) {
case DESTINATION_ATTRIBUTE:
jsonPathResults.put(jsonPathAttrKey, resultRepresentation);
@@ -232,4 +234,5 @@ public class EvaluateJsonPath extends AbstractProcessor {
processSession.transfer(flowFile, REL_MATCH);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/46bf048b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 78e1b2a..8df2de0 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -27,9 +27,11 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
-import org.apache.nifi.processor.*;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.io.OutputStreamCallback;
-import org.apache.nifi.processors.standard.util.JsonUtils;
import java.io.IOException;
import java.io.OutputStream;
@@ -44,13 +46,13 @@ import java.util.*;
+ "Each generated FlowFile is comprised of an element of the specified array and transferred to relationship 'split,' "
+ "with the original file transferred to the 'original' relationship. If the specified JsonPath is not found or "
+ "does not evaluate to an array element, the original file is routed to 'failure' and no files are generated.")
-public class SplitJson extends AbstractProcessor {
+public class SplitJson extends AbstractJsonPathProcessor {
public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder()
.name("JsonPath Expression")
.description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
.required(true)
- .addValidator(JsonUtils.JSON_PATH_VALIDATOR)
+ .addValidator(JSON_PATH_VALIDATOR)
.build();
public static final Relationship REL_ORIGINAL = new Relationship.Builder().name("original").description("The original FlowFile that was split into segments. If the FlowFile fails processing, nothing will be sent to this relationship").build();
@@ -93,7 +95,7 @@ public class SplitJson extends AbstractProcessor {
final ProcessorLog logger = getLogger();
- final DocumentContext documentContext = JsonUtils.validateAndEstablishJsonContext(processSession, original);
+ final DocumentContext documentContext = validateAndEstablishJsonContext(processSession, original);
if (documentContext == null) {
logger.error("FlowFile {} did not have valid JSON content.", new Object[]{original});
@@ -129,7 +131,7 @@ public class SplitJson extends AbstractProcessor {
split = processSession.write(split, new OutputStreamCallback() {
@Override
public void process(OutputStream out) throws IOException {
- String resultSegmentContent = JsonUtils.getResultRepresentation(resultSegment);
+ String resultSegmentContent = getResultRepresentation(resultSegment);
out.write(resultSegmentContent.getBytes(StandardCharsets.UTF_8));
}
});
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/46bf048b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
index 2174c1e..68b18b8 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -16,79 +16,19 @@
*/
package org.apache.nifi.processors.standard.util;
-import com.jayway.jsonpath.Configuration;
-import com.jayway.jsonpath.DocumentContext;
-import com.jayway.jsonpath.InvalidPathException;
-import com.jayway.jsonpath.JsonPath;
-import com.jayway.jsonpath.spi.json.JsonProvider;
import net.minidev.json.JSONValue;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.Validator;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.io.InputStreamCallback;
-import org.apache.nifi.stream.io.BufferedInputStream;
-import org.apache.nifi.util.BooleanHolder;
-import org.apache.nifi.util.ObjectHolder;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.List;
-import java.util.Map;
/**
- * Provides utilities for interacting with JSON elements and JsonPath expressions and results
+ * Provides utilities for interacting with JSON elements
*
* @see <a href="http://json.org">http://json.org</a>
- * @see <a href="https://github.com/jayway/JsonPath">https://github.com/jayway/JsonPath</a>
*/
public class JsonUtils {
- static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
-
- public static final Validator JSON_PATH_VALIDATOR = new Validator() {
- @Override
- public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
- String error = null;
- try {
- JsonPath compile = JsonPath.compile(input);
- } catch (InvalidPathException ipe) {
- error = ipe.toString();
- }
- return new ValidationResult.Builder().valid(error == null).explanation(error).build();
- }
- };
-
- public static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
-
- final BooleanHolder validJsonHolder = new BooleanHolder(false);
- processSession.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- validJsonHolder.set(JsonUtils.isValidJson(in));
- }
- });
-
- // Parse the document once into an associated context to support multiple path evaluations if specified
- final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
-
- if (validJsonHolder.get()) {
- processSession.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) {
- DocumentContext ctx = JsonPath.parse(in);
- contextHolder.set(ctx);
- }
- }
- });
- }
-
- return contextHolder.get();
- }
-
/**
* JSONValue#isValidJson is permissive to the degree of the Smart JSON definition, accordingly a strict JSON approach
* is preferred in determining whether or not a document is valid.
@@ -104,24 +44,4 @@ public class JsonUtils {
}
}
- /**
- * Determines the context by which JsonSmartJsonProvider would treat the value. {@link java.util.Map} and
- * {@link java.util.List} objects can be rendered as JSON elements, everything else is treated as a scalar.
- *
- * @param obj item to be inspected if it is a scalar or a JSON element
- * @return false, if the object is a supported type; true otherwise
- */
- public static boolean isJsonScalar(Object obj) {
- // For the default provider, JsonSmartJsonProvider, a Map or List is able to be handled as a JSON entity
- return !(obj instanceof Map || obj instanceof List);
- }
-
-
- public static String getResultRepresentation(Object jsonPathResult) {
- if (JsonUtils.isJsonScalar(jsonPathResult)) {
- return jsonPathResult.toString();
- }
- return JSON_PROVIDER.toJson(jsonPathResult);
- }
-
}
[02/50] incubator-nifi git commit: Refining logic of how errors are
handled on a per destination basis. Adding supporting tests to ensure
contract is met.
Posted by mc...@apache.org.
Refining logic of how errors are handled on a per destination basis. Adding supporting tests to ensure contract is met.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/bcebba66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/bcebba66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/bcebba66
Branch: refs/heads/NIFI-353
Commit: bcebba6632165b6d69eb88ad12895e7678079267
Parents: 5b145e1
Author: Aldrin Piri <al...@gmail.com>
Authored: Mon Feb 16 22:44:06 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Mon Feb 16 22:44:06 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 62 ++++++++++++++------
.../standard/TestEvaluateJsonPath.java | 20 ++++++-
2 files changed, 61 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bcebba66/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index d6bcdc1..b82764d 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -50,15 +50,23 @@ import java.util.*;
@SideEffectFree
@SupportsBatching
@Tags({"JSON", "evaluate", "JsonPath"})
-@CapabilityDescription("")
+@CapabilityDescription("Evaluates one or more JsonPath expressions against the content of a FlowFile. The results of those expressions are assigned to "
+ + "FlowFile Attributes or are written to the content of the FlowFile itself, depending on configuration of the "
+ + "Processor. JsonPaths are entered by adding user-defined properties; the name of the property maps to the Attribute "
+ + "Name into which the result will be placed (if the Destination is flowfile-attribute; otherwise, the property name is ignored). "
+ + "The value of the property must be a valid JsonPath expression. If the JsonPath evaluates to a JSON array or JSON object and the Return Type is "
+ + "set to 'scalar' the FlowFile will be unmodified and will be routed to failure. If the JsonPath does not "
+ + "evaluate to a scalar, the FlowFile will be routed to 'unmatched' without having its contents modified. If Destination is "
+ + "flowfile-attribute and the expression matches nothing, attributes will be created with empty strings as the value, and the "
+ + "FlowFile will always be routed to 'matched.' If Destination is 'flowfile-content' and the expression matches nothing, "
+ + "the FlowFile will be routed to 'unmatched' without having its contents modified.")
public class EvaluateJsonPath extends AbstractProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
public static final String DESTINATION_CONTENT = "flowfile-content";
- public static final String RETURN_TYPE_AUTO = "auto-detect";
public static final String RETURN_TYPE_JSON = "json";
- public static final String RETURN_TYPE_STRING = "string";
+ public static final String RETURN_TYPE_SCALAR = "scalar";
public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
.name("Destination")
@@ -72,8 +80,8 @@ public class EvaluateJsonPath extends AbstractProcessor {
.name("Return Type")
.description("Indicates the desired return type of the JSON Path expressions. Selecting 'auto-detect' will set the return type to 'json' for a Destination of 'flowfile-content', and 'string' for a Destination of 'flowfile-attribute'.")
.required(true)
- .allowableValues(RETURN_TYPE_AUTO, RETURN_TYPE_JSON, RETURN_TYPE_STRING)
- .defaultValue(RETURN_TYPE_AUTO)
+ .allowableValues(RETURN_TYPE_JSON, RETURN_TYPE_SCALAR)
+ .defaultValue(RETURN_TYPE_JSON)
.build();
public static final Relationship REL_MATCH = new Relationship.Builder().name("matched").description("FlowFiles are routed to this relationship when the JsonPath is successfully evaluated and the FlowFile is modified as a result").build();
@@ -208,30 +216,44 @@ public class EvaluateJsonPath extends AbstractProcessor {
final Map<String, String> jsonPathResults = new HashMap<>();
// Iterate through all JsonPath entries specified
+ jsonPathEvalLoop:
for (final Map.Entry<String, JsonPath> attributeJsonPathEntry : attributeToJsonPathMap.entrySet()) {
String jsonPathAttrKey = attributeJsonPathEntry.getKey();
JsonPath jsonPathExp = attributeJsonPathEntry.getValue();
- final ObjectHolder<String> resultHolder = new ObjectHolder<>("");
+ final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
try {
- resultHolder.set(evaluatePathForContext(jsonPathExp, documentContext));
+ Object result = documentContext.read(jsonPathExp);
+ if (RETURN_TYPE.getName().equals(RETURN_TYPE_SCALAR) && !isScalar(result)) {
+ logger.error("Unable to return a scalar value for a JsonPath {} for FlowFile {}. Transferring to {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), REL_FAILURE.getName()});
+ processSession.transfer(flowFile, REL_FAILURE);
+ continue flowFileLoop;
+ }
+ resultHolder.set(result);
} catch (PathNotFoundException e) {
- logger.error("FlowFile {} could not find path {} for attribute key {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), jsonPathAttrKey}, e);
- jsonPathResults.put(jsonPathAttrKey, "");
+ logger.warn("FlowFile {} could not find path {} for attribute key {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), jsonPathAttrKey}, e);
+ if (destination.equals(DESTINATION_ATTRIBUTE)) {
+ jsonPathResults.put(jsonPathAttrKey, "");
+ continue jsonPathEvalLoop;
+ } else {
+ processSession.transfer(flowFile, REL_NO_MATCH);
+ continue flowFileLoop;
+ }
}
+ final String resultRepresentation = getResultRepresentation(resultHolder.get());
switch (destination) {
case DESTINATION_ATTRIBUTE:
- jsonPathResults.put(jsonPathAttrKey, resultHolder.get());
+ jsonPathResults.put(jsonPathAttrKey, resultRepresentation);
break;
case DESTINATION_CONTENT:
flowFile = processSession.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream out) throws IOException {
try (OutputStream outputStream = new BufferedOutputStream(out)) {
- outputStream.write(resultHolder.get().getBytes(StandardCharsets.UTF_8));
+ outputStream.write(resultRepresentation.getBytes(StandardCharsets.UTF_8));
}
}
});
@@ -243,16 +265,18 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
}
- private static String evaluatePathForContext(JsonPath path, ReadContext readCtx) {
- Object pathResult = readCtx.read(path);
+ private static String getResultRepresentation(Object jsonPathResult) {
+ if (isScalar(jsonPathResult)) {
+ return jsonPathResult.toString();
+ }
+ return JSON_PROVIDER.toJson(jsonPathResult);
+ }
+
+ private static boolean isScalar(Object obj) {
/*
- * A given path could be a JSON object or a single value, if a sole value, treat as a String; otherwise, return the
- * representative JSON.
+ * A given path could be a JSON object or a single/scalar value
*/
- if (pathResult instanceof String) {
- return pathResult.toString();
- }
- return JSON_PROVIDER.toJson(pathResult);
+ return (obj instanceof String);
}
private static class JsonPathValidator implements Validator {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bcebba66/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
index bece3da..e72d069 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
@@ -180,7 +180,7 @@ public class TestEvaluateJsonPath {
@Test
- public void testExtractPath_destinationAttribute_indefiniteResult() throws Exception {
+ public void testExtractPath_destinationContent_indefiniteResult() throws Exception {
String jsonPathAttrKey = "friends.indefinite.id.list";
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
@@ -197,7 +197,7 @@ public class TestEvaluateJsonPath {
}
@Test
- public void testExtractPath_destinationAttribute_indefiniteResult_operators() throws Exception {
+ public void testExtractPath_destinationContent_indefiniteResult_operators() throws Exception {
String jsonPathAttrKey = "friends.indefinite.id.list";
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
@@ -212,4 +212,20 @@ public class TestEvaluateJsonPath {
testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
testRunner.getFlowFilesForRelationship(expectedRel).get(0).assertContentEquals("[0,1,2]");
}
+
+ @Test
+ public void testRouteUnmatched_destinationContent_noMatch() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
+ testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_CONTENT);
+ testRunner.setProperty("jsonPath", "$[0].nonexistent.key");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ Relationship expectedRel = EvaluateJsonPath.REL_NO_MATCH;
+
+ testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
+ testRunner.getFlowFilesForRelationship(expectedRel).get(0).assertContentEquals(JSON_SNIPPET);
+ }
+
}
[50/50] incubator-nifi git commit: NIFI-353: - Updating markup
generation flow. - Adding a combo box to view the content in original,
formatted, or hex form (still not functional).
Posted by mc...@apache.org.
NIFI-353:
- Updating markup generation flow.
- Adding a combo box to view the content in original, formatted, or hex form (still not functional).
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/347e4e02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/347e4e02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/347e4e02
Branch: refs/heads/NIFI-353
Commit: 347e4e024d4e990e6eb17a2101c31db79200f7a7
Parents: 37b5ca4
Author: Matt Gilman <ma...@gmail.com>
Authored: Thu Mar 5 23:20:39 2015 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Thu Mar 5 23:20:39 2015 -0500
----------------------------------------------------------------------
.../nifi/web/ContentViewerController.java | 22 +++++++-----
.../src/main/webapp/WEB-INF/jsp/header.jsp | 38 ++++++++++++++++++--
.../src/main/webapp/WEB-INF/jsp/hexview.jsp | 1 +
.../src/main/webapp/WEB-INF/web.xml | 5 +--
.../src/main/webapp/css/main.css | 24 +++++++++++++
.../webapp/js/jquery/combo/jquery.combo.css | 8 +++++
.../src/main/webapp/WEB-INF/jsp/codemirror.jsp | 2 +-
7 files changed, 85 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/java/org/apache/nifi/web/ContentViewerController.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/java/org/apache/nifi/web/ContentViewerController.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/java/org/apache/nifi/web/ContentViewerController.java
index cad4819..c5a6fb5 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/java/org/apache/nifi/web/ContentViewerController.java
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/java/org/apache/nifi/web/ContentViewerController.java
@@ -76,19 +76,26 @@ public class ContentViewerController extends HttpServlet {
mode = DisplayMode.Original.name();
}
+ // determine the display mode
final DisplayMode displayMode;
try {
displayMode = DisplayMode.valueOf(mode);
} catch (final IllegalArgumentException iae) {
- final PrintWriter out = response.getWriter();
- out.println("Invalid display mode: " + mode);
-
+ response.getWriter().println("Invalid display mode: " + mode);
return;
}
+ // build the request url
+ final String queryParams = request.getQueryString();
+ final StringBuffer requestUrl = request.getRequestURL();
+ requestUrl.append("?").append(queryParams);
+ request.setAttribute("requestUrl", requestUrl);
+
// generate the header
- final RequestDispatcher header = request.getRequestDispatcher("/WEB-INF/jsp/header.jsp");
- header.include(request, response);
+ request.getRequestDispatcher("/WEB-INF/jsp/header.jsp").include(request, response);
+
+ // remove the request url
+ request.removeAttribute("requestUrl");
// generate the markup for the content based on the display mode
if (DisplayMode.Hex.equals(displayMode)) {
@@ -128,7 +135,7 @@ public class ContentViewerController extends HttpServlet {
return;
}
-
+
// create a request attribute for accessing the content
request.setAttribute(ViewableContent.CONTENT_REQUEST_ATTRIBUTE, new ViewableContent() {
@Override
@@ -179,8 +186,7 @@ public class ContentViewerController extends HttpServlet {
}
// generate footer
- final RequestDispatcher footer = request.getRequestDispatcher("/WEB-INF/jsp/footer.jsp");
- footer.include(request, response);
+ request.getRequestDispatcher("/WEB-INF/jsp/footer.jsp").include(request, response);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/header.jsp
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/header.jsp b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/header.jsp
index 0a17604..d0323ef 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/header.jsp
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/header.jsp
@@ -25,10 +25,44 @@
<link href="../nifi/css/message-pane.css" rel="stylesheet" type="text/css" />
<link href="../nifi/css/message-page.css" rel="stylesheet" type="text/css" />
<link rel="stylesheet" href="../nifi/js/jquery/combo/jquery.combo.css" type="text/css" />
+ <link rel="stylesheet" href="../nifi/css/reset.css" type="text/css" />
<script type="text/javascript" src="../nifi/js/jquery/jquery-2.1.1.min.js"></script>
<script type="text/javascript" src="../nifi/js/jquery/combo/jquery.combo.js"></script>
- <script type="text/javascript" src="js/application.js"></script>
+ <script type="text/javascript">
+ var $$ = $.noConflict(true);
+ $$(document).ready(function () {
+ $$('#view-as').combo({
+ options: [{
+ text: 'original',
+ value: '${requestUrl}&mode=Original'
+ }, {
+ text: 'formatted',
+ value: '${requestUrl}&mode=Formatted'
+ }, {
+ text: 'hex',
+ value: '${requestUrl}&mode=Hex'
+ }],
+ select: function (option) {
+// var currentLocation = window.location.href;
+// if (option.text === 'original') {
+// if (currentLocation.indexOf('&mode=Original') === -1) {
+// window.location.href = option.value;
+// }
+// } else if (option.text === 'formatted') {
+// if (currentLocation.indexOf('&mode=Formatted') === -1) {
+// window.location.href = option.value;
+// }
+// } else if (option.text === 'hex') {
+// if (currentLocation.indexOf('&mode=Hex') === -1) {
+// window.location.href = option.value;
+// }
+// }
+ }
+ });
+ });
+ </script>
</head>
-
<body class="message-pane">
+ <div id="view-as-label">View as</div>
+ <div id="view-as" class="pointer button-normal"></div>
<div class="message-pane-message-box">
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/hexview.jsp
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/hexview.jsp b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/hexview.jsp
index f2270d7..f5981ac 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/hexview.jsp
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/jsp/hexview.jsp
@@ -16,6 +16,7 @@
--%>
<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
<link rel="stylesheet" href="js/hexview/hexview.default.css" type="text/css" />
+<script type="text/javascript" src="../nifi/js/jquery/jquery-2.1.1.min.js"></script>
<script type="text/javascript" src="js/hexview/hexview.js"></script>
<div id="hexview-content" class="hexviewerwindow">
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/web.xml b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/web.xml
index 0765a78..77f0853 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/web.xml
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/WEB-INF/web.xml
@@ -21,9 +21,6 @@
</servlet>
<servlet-mapping>
<servlet-name>ContentViewerController</servlet-name>
- <url-pattern>/viewer</url-pattern>
+ <url-pattern></url-pattern>
</servlet-mapping>
- <welcome-file-list>
- <welcome-file>viewer</welcome-file>
- </welcome-file-list>
</web-app>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/css/main.css
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/css/main.css b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/css/main.css
index dad3120..f814e55 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/css/main.css
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-content-viewer/src/main/webapp/css/main.css
@@ -17,4 +17,28 @@
* {
margin: 0;
padding: 0;
+}
+
+#view-as-label {
+ position: absolute;
+ top: 72px;
+ left: 144px;
+ line-height: 24px;
+ font-size: 12px;
+}
+
+#view-as {
+ position: absolute;
+ top: 72px;
+ left: 200px;
+ width: 80px;
+ height: 18px;
+}
+
+.pointer {
+ cursor: pointer;
+}
+
+.hidden {
+ display: none;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
index 0432340..edf66f3 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/combo/jquery.combo.css
@@ -86,3 +86,11 @@ div.combo-nifi-tooltip {
background-color: #FFFFA3;
color: #454545;
}
+
+div.button-normal {
+ background: transparent url(../../../images/bgButton.png) repeat-x center center;
+}
+
+div.button-over {
+ background: transparent url(../../../images/bgButtonOver.png) repeat-x center center;
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/347e4e02/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/webapp/WEB-INF/jsp/codemirror.jsp
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/webapp/WEB-INF/jsp/codemirror.jsp b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/webapp/WEB-INF/jsp/codemirror.jsp
index 3a257c8..65ad826 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/webapp/WEB-INF/jsp/codemirror.jsp
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/webapp/WEB-INF/jsp/codemirror.jsp
@@ -31,7 +31,7 @@
matchBrackets: true,
foldGutter: true,
gutters: ["CodeMirror-linenumbers", "CodeMirror-foldgutter"],
- readOnly: true,
+ readOnly: true
});
var setEditorSize = function() {
[12/50] incubator-nifi git commit: Preferring the constant empty
string for clarity instead of quotes.
Posted by mc...@apache.org.
Preferring the constant empty string for clarity instead of quotes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/408f6363
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/408f6363
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/408f6363
Branch: refs/heads/NIFI-353
Commit: 408f636361cbad025752978e10775651b7daddf4
Parents: a058fd0
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 17:01:06 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 17:01:06 2015 -0500
----------------------------------------------------------------------
.../org/apache/nifi/processors/standard/EvaluateJsonPath.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/408f6363/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 093454d..3a64a34 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -35,6 +35,7 @@ import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processors.standard.util.JsonUtils;
import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.util.ObjectHolder;
+import org.apache.nifi.util.StringUtils;
import java.io.IOException;
import java.io.OutputStream;
@@ -206,7 +207,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
} catch (PathNotFoundException e) {
logger.warn("FlowFile {} could not find path {} for attribute key {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), jsonPathAttrKey}, e);
if (destination.equals(DESTINATION_ATTRIBUTE)) {
- jsonPathResults.put(jsonPathAttrKey, "");
+ jsonPathResults.put(jsonPathAttrKey, StringUtils.EMPTY);
continue jsonPathEvalLoop;
} else {
processSession.transfer(flowFile, REL_NO_MATCH);
[10/50] incubator-nifi git commit: Adding an auto return type for
EvaluateJsonPath to match the semantics of the EvaluateXPath processor.
Posted by mc...@apache.org.
Adding an auto return type for EvaluateJsonPath to match the semantics of the EvaluateXPath processor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/627bd91f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/627bd91f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/627bd91f
Branch: refs/heads/NIFI-353
Commit: 627bd91fa719748b850b6c610a89b44be3e6f35e
Parents: 1a74621
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 16:15:33 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 16:15:33 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/processors/standard/EvaluateJsonPath.java | 10 +++++++---
.../nifi/processors/standard/TestEvaluateJsonPath.java | 1 +
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/627bd91f/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 8cdf1a2..093454d 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -61,6 +61,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
public static final String DESTINATION_CONTENT = "flowfile-content";
+ public static final String RETURN_TYPE_AUTO = "auto-detect";
public static final String RETURN_TYPE_JSON = "json";
public static final String RETURN_TYPE_SCALAR = "scalar";
@@ -76,8 +77,8 @@ public class EvaluateJsonPath extends AbstractProcessor {
.name("Return Type")
.description("Indicates the desired return type of the JSON Path expressions. Selecting 'auto-detect' will set the return type to 'json' for a Destination of 'flowfile-content', and 'string' for a Destination of 'flowfile-attribute'.")
.required(true)
- .allowableValues(RETURN_TYPE_JSON, RETURN_TYPE_SCALAR)
- .defaultValue(RETURN_TYPE_JSON)
+ .allowableValues(RETURN_TYPE_AUTO, RETURN_TYPE_JSON, RETURN_TYPE_SCALAR)
+ .defaultValue(RETURN_TYPE_AUTO)
.build();
public static final Relationship REL_MATCH = new Relationship.Builder().name("matched").description("FlowFiles are routed to this relationship when the JsonPath is successfully evaluated and the FlowFile is modified as a result").build();
@@ -168,7 +169,10 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
final String destination = processContext.getProperty(DESTINATION).getValue();
- final String returnType = processContext.getProperty(RETURN_TYPE).getValue();
+ String returnType = processContext.getProperty(RETURN_TYPE).getValue();
+ if (returnType.equals(RETURN_TYPE_AUTO)) {
+ returnType = destination.equals(DESTINATION_CONTENT) ? RETURN_TYPE_JSON : RETURN_TYPE_SCALAR;
+ }
flowFileLoop:
for (FlowFile flowFile : flowFiles) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/627bd91f/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
index b7b5103..c5ff814 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
@@ -100,6 +100,7 @@ public class TestEvaluateJsonPath {
public void testExtractPath_destinationAttributes_twoPaths() throws Exception {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_ATTRIBUTE);
+ testRunner.setProperty(EvaluateJsonPath.RETURN_TYPE, EvaluateJsonPath.RETURN_TYPE_JSON);
String jsonPathIdAttrKey = "evaluatejson.id";
String jsonPathNameAttrKey = "evaluatejson.name";
[33/50] incubator-nifi git commit: Adding notes about JsonPath
loading contents into memory for both JsonPath processors.
Posted by mc...@apache.org.
Adding notes about JsonPath loading contents into memory for both JsonPath processors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/5a2a8fc6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/5a2a8fc6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/5a2a8fc6
Branch: refs/heads/NIFI-353
Commit: 5a2a8fc6befb8305407a3f9b90443b66237e14f5
Parents: 84602ca
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 13:37:29 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 13:37:29 2015 -0500
----------------------------------------------------------------------
.../index.html | 5 +++++
.../org.apache.nifi.processors.standard.SplitJson/index.html | 6 ++++++
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5a2a8fc6/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
index 055a794..58da666 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
@@ -36,7 +36,12 @@
evaluate to a defined path, the FlowFile will be routed to 'unmatched' without having its contents modified. If
Destination is flowfile-attribute and the expression matches nothing, attributes will be created with empty
strings as the value, and the FlowFile will always be routed to 'matched.'
+</p>
+<p>
+ <strong>Note:</strong> The underlying JsonPath library loads the entirety of the streamed content into and performs
+ result evaluations in memory. Accordingly, it is important to consider the anticipated profile of content being
+ evaluated by this processor and the hardware supporting it especially when working against large JSON documents.
</p>
<p>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5a2a8fc6/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
index 6fc8e61..e3a4b14 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
@@ -33,6 +33,12 @@
does not evaluate to an array element, the original file is routed to 'failure' and no files are generated.
</p>
+<p>
+ <strong>Note:</strong> The underlying JsonPath library loads the entirety of the streamed content into and performs
+ result evaluations in memory. Accordingly, it is important to consider the anticipated profile of content being
+ evaluated by this processor and the hardware supporting it especially when working against large JSON documents.
+</p>
+
<strong>Properties:</strong>
</p>
[49/50] incubator-nifi git commit: Merge branch 'develop' into
NIFI-353
Posted by mc...@apache.org.
Merge branch 'develop' into NIFI-353
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/37b5ca48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/37b5ca48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/37b5ca48
Branch: refs/heads/NIFI-353
Commit: 37b5ca48f43cd3bb80b080e14c7cc2478da859aa
Parents: 4819228 5e0026c
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Mar 4 21:44:10 2015 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Mar 4 21:44:10 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 1 -
nifi/NOTICE | 5 +
nifi/nifi-assembly/LICENSE | 28 +
nifi/nifi-assembly/NOTICE | 8 +
nifi/nifi-assembly/pom.xml | 5 +
.../nifi/util/MockControllerServiceLookup.java | 3 +
.../apache/nifi/util/MockProcessContext.java | 4 +-
.../repository/StandardProcessSession.java | 46 +-
.../src/main/resources/META-INF/LICENSE | 29 +
.../src/main/resources/META-INF/NOTICE | 10 +-
.../nifi-standard-processors/pom.xml | 13 +
.../standard/AbstractJsonPathProcessor.java | 117 ++++
.../nifi/processors/standard/EncodeContent.java | 248 ++++++++
.../processors/standard/EvaluateJsonPath.java | 278 +++++++++
.../processors/standard/HandleHttpRequest.java | 604 +++++++++++++++++++
.../processors/standard/HandleHttpResponse.java | 178 ++++++
.../processors/standard/IdentifyMimeType.java | 405 ++-----------
.../nifi/processors/standard/SplitJson.java | 184 ++++++
.../util/JsonPathExpressionValidator.java | 487 +++++++++++++++
.../util/ValidatingBase32InputStream.java | 78 +++
.../org.apache.nifi.processor.Processor | 4 +
.../index.html | 155 +++++
.../index.html | 255 ++++++++
.../index.html | 112 ++++
.../index.html | 85 +++
.../org/apache/tika/mime/custom-mimetypes.xml | 83 +++
.../processors/standard/TestEncodeContent.java | 164 +++++
.../standard/TestEvaluateJsonPath.java | 264 ++++++++
.../standard/TestHandleHttpRequest.java | 128 ++++
.../standard/TestHandleHttpResponse.java | 172 ++++++
.../standard/TestIdentifyMimeType.java | 80 +--
.../nifi/processors/standard/TestSplitJson.java | 126 ++++
.../test/resources/TestIdentifyMimeType/1.tar | Bin 2048 -> 10240 bytes
.../resources/TestIdentifyMimeType/1.tar.gz | Bin 0 -> 154 bytes
.../test/resources/TestIdentifyMimeType/1.txt | 1 +
.../test/resources/TestJson/json-sample.json | 415 +++++++++++++
.../nifi-http-context-map-api/pom.xml | 23 +
.../org/apache/nifi/http/HttpContextMap.java | 72 +++
.../nifi-http-context-map-nar/pom.xml | 37 ++
.../nifi-http-context-map/pom.xml | 44 ++
.../nifi/http/StandardHttpContextMap.java | 176 ++++++
...org.apache.nifi.controller.ControllerService | 1 +
.../index.html | 67 ++
.../nifi-http-context-map-bundle/pom.xml | 31 +
.../nifi-standard-services-api-nar/pom.xml | 5 +
.../nifi-standard-services/pom.xml | 2 +
nifi/nifi-nar-bundles/pom.xml | 11 +
nifi/pom.xml | 11 +
48 files changed, 4818 insertions(+), 437 deletions(-)
----------------------------------------------------------------------
[36/50] incubator-nifi git commit: NIFI-296: Use only tika-core to
keep jar footprint down
Posted by mc...@apache.org.
NIFI-296: Use only tika-core to keep jar footprint down
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/089eec2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/089eec2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/089eec2e
Branch: refs/heads/NIFI-353
Commit: 089eec2e0c03d0d3546c36fba716c71f48f66368
Parents: b418b89
Author: Adam Lamar <ad...@gmail.com>
Authored: Tue Feb 24 20:26:13 2015 +0000
Committer: Adam Lamar <ad...@gmail.com>
Committed: Sun Mar 1 19:58:46 2015 +0000
----------------------------------------------------------------------
.../nifi-standard-processors/pom.xml | 6 --
.../processors/standard/IdentifyMimeType.java | 86 ++++---------------
.../org/apache/tika/mime/custom-mimetypes.xml | 83 ++++++++++++++++++
.../standard/TestIdentifyMimeType.java | 24 ++++++
.../resources/TestIdentifyMimeType/1.tar.gz | Bin 0 -> 154 bytes
5 files changed, 123 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/089eec2e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index b941d03..e31f0fa 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -159,12 +159,6 @@
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>1.7</version>
- <type>pom</type>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parsers</artifactId>
- <version>1.7</version>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/089eec2e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
index fd3d4ae..68880e6 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
@@ -19,14 +19,9 @@ package org.apache.nifi.processors.standard;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
@@ -42,15 +37,9 @@ import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.processor.io.InputStreamCallback;
-import org.apache.nifi.stream.io.StreamUtils;
-import org.apache.nifi.util.FlowFilePackagerV1;
-import org.apache.nifi.util.FlowFilePackagerV3;
import org.apache.nifi.util.ObjectHolder;
import org.apache.tika.config.TikaConfig;
-import org.apache.tika.detect.CompositeDetector;
-import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
-import org.apache.tika.detect.MagicDetector;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -97,16 +86,7 @@ public class IdentifyMimeType extends AbstractProcessor {
public IdentifyMimeType() {
// Setup Tika
this.config = TikaConfig.getDefaultConfig();
- DefaultDetector ddetector = new DefaultDetector();
-
- // Create list of detectors, preferring our custom detectors first
- List<Detector> detectors = new ArrayList<>();
- detectors.add(getFlowFileV3Detector());
- detectors.add(getFlowFileV1Detector());
- detectors.addAll(ddetector.getDetectors());
-
- CompositeDetector compositeDetector = new CompositeDetector(detectors);
- this.detector = compositeDetector;
+ this.detector = config.getDetector();
}
@Override
@@ -129,9 +109,8 @@ public class IdentifyMimeType extends AbstractProcessor {
}
final ProcessorLog logger = getLogger();
-
final ObjectHolder<String> mimeTypeRef = new ObjectHolder<>(null);
- final ObjectHolder<String> extensionRef = new ObjectHolder<>(null);
+
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream stream) throws IOException {
@@ -141,20 +120,25 @@ public class IdentifyMimeType extends AbstractProcessor {
// Get mime type
MediaType mediatype = detector.detect(tikaStream, metadata);
mimeTypeRef.set(mediatype.toString());
- // Get common file extension
- try {
- MimeType mimetype;
- mimetype = config.getMimeRepository().forName(mediatype.toString());
- extensionRef.set(mimetype.getExtension());
- } catch (MimeTypeException ex) {
- logger.warn("MIME type detection failed: {}", new Object[]{ex});
- }
}
}
});
String mimeType = mimeTypeRef.get();
- String extension = extensionRef.get();
+ String extension = "";
+ try {
+ MimeType mimetype;
+ mimetype = config.getMimeRepository().forName(mimeType);
+ extension = mimetype.getExtension();
+ } catch (MimeTypeException ex) {
+ logger.warn("MIME type extension lookup failed: {}", new Object[]{ex});
+ }
+
+ // Workaround for bug in Tika - https://issues.apache.org/jira/browse/TIKA-1563
+ if (mimeType != null && mimeType.equals("application/gzip") && extension.equals(".tgz")) {
+ extension = ".gz";
+ }
+
if (mimeType == null) {
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/octet-stream");
flowFile = session.putAttribute(flowFile, "mime.extension", "");
@@ -168,42 +152,4 @@ public class IdentifyMimeType extends AbstractProcessor {
session.getProvenanceReporter().modifyAttributes(flowFile);
session.transfer(flowFile, REL_SUCCESS);
}
-
- private Detector getFlowFileV3Detector() {
- return new MagicDetector(FLOWFILE_V3, FlowFilePackagerV3.MAGIC_HEADER);
- }
-
- private Detector getFlowFileV1Detector() {
- return new FlowFileV1Detector();
- }
-
- private class FlowFileV1Detector implements Detector {
-
- @Override
- public MediaType detect(InputStream in, Metadata mtdt) throws IOException {
- // Sanity check the stream. This may not be a tarfile at all
- in.mark(FlowFilePackagerV1.FILENAME_ATTRIBUTES.length());
- byte[] bytes = new byte[FlowFilePackagerV1.FILENAME_ATTRIBUTES.length()];
- StreamUtils.fillBuffer(in, bytes, false);
- in.reset();
-
- // Quick exit if the first filename is not correct
- if (!Arrays.equals(bytes, FlowFilePackagerV1.FILENAME_ATTRIBUTES.getBytes())) {
- return MediaType.OCTET_STREAM;
- }
-
- // More in-depth detection
- final TarArchiveInputStream tarIn = new TarArchiveInputStream(in);
- final TarArchiveEntry firstEntry = tarIn.getNextTarEntry();
- if (firstEntry != null) {
- if (firstEntry.getName().equals(FlowFilePackagerV1.FILENAME_ATTRIBUTES)) {
- final TarArchiveEntry secondEntry = tarIn.getNextTarEntry();
- if (secondEntry != null && secondEntry.getName().equals(FlowFilePackagerV1.FILENAME_CONTENT)) {
- return FLOWFILE_V1;
- }
- }
- }
- return MediaType.OCTET_STREAM;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/089eec2e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml
new file mode 100644
index 0000000..657b4b5
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<mime-info>
+
+ <mime-type type="application/flowfile-v1">
+ <_comment>NiFi FlowFile V1</_comment>
+ <sub-class-of type="application/x-tar"/>
+ <magic>
+ <match value="flowfile.attributes" type="string" offset="0" />
+ </magic>
+ </mime-type>
+
+ <mime-type type="application/flowfile-v3">
+ <_comment>NiFi FlowFile V3</_comment>
+ <magic>
+ <match value="NiFiFF3" type="string" offset="0" />
+ </magic>
+ </mime-type>
+
+ <mime-type type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet">
+ <_comment>Office Open XML Workbook</_comment>
+ <glob pattern="*.xlsx"/>
+ <sub-class-of type="application/x-tika-ooxml"/>
+ <magic priority="60">
+ <match value="PK\003\004" type="string" offset="0">
+ <match value="[Content_Types].xml" type="string" offset="30">
+ <match value="xl/_rels/workbook.xml.rels" type="string" offset="30:4096"/>
+ </match>
+ </match>
+ </magic>
+ </mime-type>
+
+ <mime-type type="application/vnd.openxmlformats-officedocument.wordprocessingml.document">
+ <_comment>Office Open XML Document</_comment>
+ <glob pattern="*.docx"/>
+ <sub-class-of type="application/x-tika-ooxml"/>
+ <magic priority="60">
+ <match value="PK\003\004" type="string" offset="0">
+ <match value="[Content_Types].xml" type="string" offset="30">
+ <match value="word/_rels/document.xml.rels" type="string" offset="30:4096"/>
+ </match>
+ </match>
+ </magic>
+ </mime-type>
+
+ <mime-type type="application/vnd.openxmlformats-officedocument.presentationml.presentation">
+ <_comment>Office Open XML Presentation</_comment>
+ <glob pattern="*.pptx"/>
+ <glob pattern="*.thmx"/>
+ <sub-class-of type="application/x-tika-ooxml"/>
+ <magic priority="60">
+ <match value="PK\003\004" type="string" offset="0">
+ <match value="[Content_Types].xml" type="string" offset="30">
+ <match value="ppt/slides/_rels/slide" type="string" offset="30:4096"/>
+ </match>
+ </match>
+ </magic>
+ </mime-type>
+
+ <mime-type type="application/java-archive">
+ <_comment>Java Archive</_comment>
+ <tika:link>http://en.wikipedia.org/wiki/.jar</tika:link>
+ <tika:uti>com.sun.java-archive</tika:uti>
+ <sub-class-of type="application/zip"/>
+ <glob pattern="*.jar"/>
+ <magic priority="50">
+ <match value="PK\003\004" type="string" offset="0">
+ <match value="META-INF/MANIFEST.MF" type="string" offset="0:1024"/>
+ </match>
+ </magic>
+ </mime-type>
+
+ <!-- Override tika's default behavior for GNU tar detection because nobody calls
+ a GNU tar a .gtar -->
+ <mime-type type="application/x-tar">
+ <_comment>GNU tar Compressed File Archive (GNU Tape Archive)</_comment>
+ <magic priority="60">
+ <!-- GNU tar archive -->
+ <match value="ustar \0" type="string" offset="257" />
+ </magic>
+ <glob pattern="*.tar"/>
+ </mime-type>
+
+</mime-info>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/089eec2e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
index 40b03b4..1bf4585 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
@@ -67,17 +67,41 @@ public class TestIdentifyMimeType {
expectedMimeTypes.put("1.pdf", "application/pdf");
expectedMimeTypes.put("grid.gif", "image/gif");
expectedMimeTypes.put("1.tar", "application/x-tar");
+ expectedMimeTypes.put("1.tar.gz", "application/gzip");
expectedMimeTypes.put("1.jar", "application/java-archive");
expectedMimeTypes.put("1.xml", "application/xml");
expectedMimeTypes.put("flowfilev3", "application/flowfile-v3");
expectedMimeTypes.put("flowfilev1.tar", "application/flowfile-v1");
+ final Map<String, String> expectedExtensions = new HashMap<>();
+ expectedExtensions.put("1.7z", ".7z");
+ expectedExtensions.put("1.mdb", ".mdb");
+ expectedExtensions.put("1.txt", ".txt");
+ expectedExtensions.put("1.txt.bz2", ".bz2");
+ expectedExtensions.put("1.txt.gz", ".gz");
+ expectedExtensions.put("1.zip", ".zip");
+ expectedExtensions.put("bgBannerFoot.png", ".png");
+ expectedExtensions.put("blueBtnBg.jpg", ".jpg");
+ expectedExtensions.put("1.pdf", ".pdf");
+ expectedExtensions.put("grid.gif", ".gif");
+ expectedExtensions.put("1.tar", ".tar");
+ expectedExtensions.put("1.tar.gz", ".gz");
+ expectedExtensions.put("1.jar", ".jar");
+ expectedExtensions.put("1.xml", ".xml");
+ expectedExtensions.put("flowfilev3", "");
+ expectedExtensions.put("flowfilev1.tar", "");
+
final List<MockFlowFile> filesOut = runner.getFlowFilesForRelationship(IdentifyMimeType.REL_SUCCESS);
for (final MockFlowFile file : filesOut) {
final String filename = file.getAttribute(CoreAttributes.FILENAME.key());
final String mimeType = file.getAttribute(CoreAttributes.MIME_TYPE.key());
final String expected = expectedMimeTypes.get(filename);
+
+ final String extension = file.getAttribute("mime.extension");
+ final String expectedExtension = expectedExtensions.get(filename);
+
assertEquals("Expected " + file + " to have MIME Type " + expected + ", but it was " + mimeType, expected, mimeType);
+ assertEquals("Expected " + file + " to have extension " + expectedExtension + ", but it was " + extension, expectedExtension, extension);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/089eec2e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar.gz
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar.gz b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar.gz
new file mode 100755
index 0000000..481ccc1
Binary files /dev/null and b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar.gz differ
[04/50] incubator-nifi git commit: Moving the JsonPath Validator to
the JsonPathUtil class so that it can be reused by other processors.
Posted by mc...@apache.org.
Moving the JsonPath Validator to the JsonPathUtil class so that it can be reused by other processors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/5a81f19b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/5a81f19b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/5a81f19b
Branch: refs/heads/NIFI-353
Commit: 5a81f19b25bc073ec1089c115b6e6fcf01056f91
Parents: 0a19ada
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 09:44:30 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 09:44:30 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 21 +++-------
.../processors/standard/util/JsonPathUtils.java | 44 ++++++++++++++++++++
2 files changed, 50 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5a81f19b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index fe89635..05d31da 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -16,7 +16,10 @@
*/
package org.apache.nifi.processors.standard;
-import com.jayway.jsonpath.*;
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
import com.jayway.jsonpath.spi.json.JsonProvider;
import net.minidev.json.JSONValue;
import org.apache.nifi.annotation.behavior.EventDriven;
@@ -27,13 +30,13 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.Validator;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.*;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processors.standard.util.JsonPathUtils;
import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.util.BooleanHolder;
@@ -147,7 +150,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.expressionLanguageSupported(false)
- .addValidator(new JsonPathValidator())
+ .addValidator(JsonPathUtils.JSON_PATH_VALIDATOR)
.required(false)
.dynamic(true)
.build();
@@ -278,17 +281,5 @@ public class EvaluateJsonPath extends AbstractProcessor {
return (obj instanceof String);
}
- private static class JsonPathValidator implements Validator {
- @Override
- public ValidationResult validate(String subject, String input, ValidationContext context) {
- String error = null;
- try {
- JsonPath compile = JsonPath.compile(input);
- } catch (InvalidPathException ipe) {
- error = ipe.toString();
- }
- return new ValidationResult.Builder().valid(error == null).explanation(error).build();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5a81f19b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java
new file mode 100644
index 0000000..ecd2ad0
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard.util;
+
+import com.jayway.jsonpath.InvalidPathException;
+import com.jayway.jsonpath.JsonPath;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+
+/**
+ * Provides utilities for interacting with JsonPath expressions and results
+ *
+ * @see <a href="https://github.com/jayway/JsonPath">https://github.com/jayway/JsonPath</a>
+ */
+public class JsonPathUtils {
+
+ public static final Validator JSON_PATH_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ String error = null;
+ try {
+ JsonPath compile = JsonPath.compile(input);
+ } catch (InvalidPathException ipe) {
+ error = ipe.toString();
+ }
+ return new ValidationResult.Builder().valid(error == null).explanation(error).build();
+ }
+ };
+}
[32/50] incubator-nifi git commit: Removing extraneous logging
statement.
Posted by mc...@apache.org.
Removing extraneous logging statement.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/84602ca3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/84602ca3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/84602ca3
Branch: refs/heads/NIFI-353
Commit: 84602ca3e9935da74788aa893998efd2685dc873
Parents: 4d3cff3
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 13:27:40 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 13:27:40 2015 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/standard/SplitJson.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/84602ca3/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 8c7ae4d..59f4d71 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -131,7 +131,6 @@ public class SplitJson extends AbstractJsonPathProcessor {
String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
final JsonPath jsonPath = JSON_PATH_MAP.get(jsonPathExpression);
- getLogger().info("Using value {} for split ", new Object[]{jsonPathExpression});
final List<FlowFile> segments = new ArrayList<>();
[43/50] incubator-nifi git commit: Merge branch 'NIFI-360' of
https://github.com/apiri/incubator-nifi into develop
Posted by mc...@apache.org.
Merge branch 'NIFI-360' of https://github.com/apiri/incubator-nifi into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/245ec0ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/245ec0ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/245ec0ca
Branch: refs/heads/NIFI-353
Commit: 245ec0ca80c6bfec71b61282d47006e41e75c445
Parents: 96a42cd e6ebaa4
Author: joewitt <jo...@apache.org>
Authored: Sun Mar 1 22:19:27 2015 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sun Mar 1 22:19:27 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 21 +
nifi/nifi-assembly/LICENSE | 28 ++
nifi/nifi-assembly/NOTICE | 8 +
.../src/main/resources/META-INF/LICENSE | 29 ++
.../src/main/resources/META-INF/NOTICE | 10 +-
.../nifi-standard-processors/pom.xml | 4 +
.../standard/AbstractJsonPathProcessor.java | 117 +++++
.../processors/standard/EvaluateJsonPath.java | 278 +++++++++++
.../nifi/processors/standard/SplitJson.java | 184 +++++++
.../util/JsonPathExpressionValidator.java | 487 +++++++++++++++++++
.../org.apache.nifi.processor.Processor | 2 +
.../index.html | 155 ++++++
.../index.html | 85 ++++
.../standard/TestEvaluateJsonPath.java | 264 ++++++++++
.../nifi/processors/standard/TestSplitJson.java | 126 +++++
.../test/resources/TestJson/json-sample.json | 415 ++++++++++++++++
nifi/pom.xml | 5 +
17 files changed, 2217 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/245ec0ca/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/245ec0ca/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/245ec0ca/nifi/pom.xml
----------------------------------------------------------------------
[44/50] incubator-nifi git commit: NIFI-360 removed extra LICENSE
information and updated NOTICE to reflect source/copyright from Json Path
Posted by mc...@apache.org.
NIFI-360 removed extra LICENSE information and updated NOTICE to reflect source/copyright from Json Path
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/67a4faf1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/67a4faf1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/67a4faf1
Branch: refs/heads/NIFI-353
Commit: 67a4faf1353530aae7828252fb474e6117e13bcc
Parents: 245ec0c
Author: joewitt <jo...@apache.org>
Authored: Sun Mar 1 23:01:31 2015 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sun Mar 1 23:01:31 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 22 ----------------------
nifi/NOTICE | 5 +++++
2 files changed, 5 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/67a4faf1/nifi/LICENSE
----------------------------------------------------------------------
diff --git a/nifi/LICENSE b/nifi/LICENSE
index d00d450..50150e3 100644
--- a/nifi/LICENSE
+++ b/nifi/LICENSE
@@ -455,25 +455,3 @@ This product bundles 'json2.js' which is available in the 'public domain'.
This product bundles 'reset.css' which is available in the 'public domain'.
For details see http://meyerweb.com/eric/tools/css/reset/
-This product bundles source from 'JsonPath'. Specifically the 'PathCompiler'.
-The source is available under an Apache Software License 2.0.
-
- Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/67a4faf1/nifi/NOTICE
----------------------------------------------------------------------
diff --git a/nifi/NOTICE b/nifi/NOTICE
index 9686fba..ba32a1b 100644
--- a/nifi/NOTICE
+++ b/nifi/NOTICE
@@ -4,3 +4,8 @@ Copyright 2014-2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+The source release of this product bundles a modified
+version of PathCompiler from https://github.com/jayway/JsonPath
+The following notice information applies
+ Copyright 2011 Kalle Stenflo, Jochen Berger
+
[41/50] incubator-nifi git commit: Adding JsonPathExpressionValidator
to perform an exception free validation of JsonPath expressions. This is used
as a screen before attempting a compile.
Posted by mc...@apache.org.
Adding JsonPathExpressionValidator to perform an exception free validation of JsonPath expressions. This is used as a screen before attempting a compile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/4618f46f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/4618f46f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/4618f46f
Branch: refs/heads/NIFI-353
Commit: 4618f46f278c50238ec98bf7021337564e641de0
Parents: 973b493
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 21:23:01 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 21:23:01 2015 -0500
----------------------------------------------------------------------
.../standard/AbstractJsonPathProcessor.java | 14 +-
.../nifi/processors/standard/SplitJson.java | 2 +-
.../util/JsonPathExpressionValidator.java | 487 +++++++++++++++++++
.../standard/TestEvaluateJsonPath.java | 12 +
4 files changed, 507 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4618f46f/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
index febc3f8..94a299e 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -18,7 +18,6 @@ package org.apache.nifi.processors.standard;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
-import com.jayway.jsonpath.InvalidPathException;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.internal.spi.json.JsonSmartJsonProvider;
import com.jayway.jsonpath.spi.json.JsonProvider;
@@ -30,6 +29,7 @@ import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processors.standard.util.JsonPathExpressionValidator;
import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.util.ObjectHolder;
@@ -90,15 +90,14 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
@Override
public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
- JsonPath compiledJsonPath = null;
String error = null;
- try {
- if (isStale(subject, input)) {
- compiledJsonPath = JsonPath.compile(input);
+ if (isStale(subject, input)) {
+ if (JsonPathExpressionValidator.isValidExpression(input)) {
+ JsonPath compiledJsonPath = JsonPath.compile(input);
cacheComputedValue(subject, input, compiledJsonPath);
+ } else {
+ error = "specified expression was not valid: " + input;
}
- } catch (InvalidPathException ipe) {
- error = ipe.toString();
}
return new ValidationResult.Builder().subject(subject).valid(error == null).explanation(error).build();
}
@@ -106,6 +105,7 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
/**
* An optional hook to act on the compute value
*/
+
abstract void cacheComputedValue(String subject, String input, JsonPath computedJsonPath);
/**
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4618f46f/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 5a193a1..4d79746 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -57,7 +57,7 @@ public class SplitJson extends AbstractJsonPathProcessor {
public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder()
.name("JsonPath Expression")
.description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) // Full validation/caching occurs in #customValidate
.required(true)
.build();
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4618f46f/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java
new file mode 100644
index 0000000..61f9bbe
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathExpressionValidator.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard.util;
+
+import com.jayway.jsonpath.Filter;
+import com.jayway.jsonpath.Predicate;
+import com.jayway.jsonpath.internal.Utils;
+import com.jayway.jsonpath.internal.token.*;
+import org.apache.nifi.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static java.util.Arrays.asList;
+
+/**
+ * JsonPathExpressionValidator performs the same execution as com.jayway.jsonpath.internal.PathCompiler, but does not throw
+ * exceptions when an invalid path segment is found.
+ * Limited access to create JsonPath objects requires a separate flow of execution in avoiding exceptions.
+ *
+ * @see <a href="https://github.com/jayway/JsonPath">https://github.com/jayway/JsonPath</a>
+ */
+public class JsonPathExpressionValidator {
+
+ private static final String PROPERTY_OPEN = "['";
+ private static final String PROPERTY_CLOSE = "']";
+ private static final char DOCUMENT = '$';
+ private static final char ANY = '*';
+ private static final char PERIOD = '.';
+ private static final char BRACKET_OPEN = '[';
+ private static final char BRACKET_CLOSE = ']';
+ private static final char SPACE = ' ';
+
+
+ /**
+ * Performs a validation of a provided JsonPath expression.
+ * <p/>
+ * Typically this is used in the context of:
+ * <code>
+ * <pre>
+ * JsonPath compiledJsonPath = null;
+ * if (JsonPathExpressionValidator.isValidExpression(input)) {
+ * compiledJsonPath = JsonPath.compile(input);
+ * ...
+ * } else {
+ * // error handling
+ * }
+ * </pre>
+ * </code>
+ *
+ * @param path to evaluate for validity
+ * @param filters applied to path expression; this is typically unused in the context of Processors
+ * @return true if the specified path is valid; false otherwise
+ */
+ public static boolean isValidExpression(String path, Predicate... filters) {
+ path = path.trim();
+ if (StringUtils.isBlank(path)) {
+ // "Path may not be null empty"
+ return false;
+ }
+ if (path.endsWith("..")) {
+ // "A path can not end with a scan."
+ return false;
+ }
+
+ LinkedList<Predicate> filterList = new LinkedList<Predicate>(asList(filters));
+
+ if (path.charAt(0) != '$' && path.charAt(0) != '@') {
+ path = "$." + path;
+ }
+
+ if (path.charAt(0) == '@') {
+ path = "$" + path.substring(1);
+ }
+
+ if (path.length() > 1 && path.charAt(1) != '.' && path.charAt(1) != '[') {
+ // "Invalid path " + path
+ return false;
+ }
+
+ RootPathToken root = null;
+
+ int i = 0;
+ int positions;
+ String fragment = "";
+
+ do {
+ char current = path.charAt(i);
+
+ switch (current) {
+ case SPACE:
+ // "Space not allowed in path"
+ return false;
+ case DOCUMENT:
+ fragment = "$";
+ i++;
+ break;
+ case BRACKET_OPEN:
+ positions = fastForwardUntilClosed(path, i);
+ fragment = path.substring(i, i + positions);
+ i += positions;
+ break;
+ case PERIOD:
+ i++;
+ if (path.charAt(i) == PERIOD) {
+ //This is a deep scan
+ fragment = "..";
+ i++;
+ } else {
+ positions = fastForward(path, i);
+ if (positions == 0) {
+ continue;
+
+ } else if (positions == 1 && path.charAt(i) == '*') {
+ fragment = new String("[*]");
+ } else {
+ fragment = PROPERTY_OPEN + path.substring(i, i + positions) + PROPERTY_CLOSE;
+ }
+ i += positions;
+ }
+ break;
+ case ANY:
+ fragment = new String("[*]");
+ i++;
+ break;
+ default:
+ positions = fastForward(path, i);
+
+ fragment = PROPERTY_OPEN + path.substring(i, i + positions) + PROPERTY_CLOSE;
+ i += positions;
+ break;
+ }
+
+ /*
+ * Analyze each component represented by a fragment. If there is a failure to properly evaluate,
+ * a null result is returned
+ */
+ PathToken analyzedComponent = PathComponentAnalyzer.analyze(fragment, filterList);
+ if (analyzedComponent == null) {
+ return false;
+ }
+
+ if (root == null) {
+ root = (RootPathToken) analyzedComponent;
+ } else {
+ root.append(analyzedComponent);
+ }
+
+
+ } while (i < path.length());
+
+ return true;
+ }
+
+ private static int fastForward(String s, int index) {
+ int skipCount = 0;
+ while (index < s.length()) {
+ char current = s.charAt(index);
+ if (current == PERIOD || current == BRACKET_OPEN || current == SPACE) {
+ break;
+ }
+ index++;
+ skipCount++;
+ }
+ return skipCount;
+ }
+
+ private static int fastForwardUntilClosed(String s, int index) {
+ int skipCount = 0;
+ int nestedBrackets = 0;
+
+ //First char is always '[' no need to check it
+ index++;
+ skipCount++;
+
+ while (index < s.length()) {
+ char current = s.charAt(index);
+
+ index++;
+ skipCount++;
+
+ if (current == BRACKET_CLOSE && nestedBrackets == 0) {
+ break;
+ }
+ if (current == BRACKET_OPEN) {
+ nestedBrackets++;
+ }
+ if (current == BRACKET_CLOSE) {
+ nestedBrackets--;
+ }
+ }
+ return skipCount;
+ }
+
+ static class PathComponentAnalyzer {
+
+ private static final Pattern FILTER_PATTERN = Pattern.compile("^\\[\\s*\\?\\s*[,\\s*\\?]*?\\s*]$"); //[?] or [?, ?, ...]
+ private int i;
+ private char current;
+
+ private final LinkedList<Predicate> filterList;
+ private final String pathFragment;
+
+ PathComponentAnalyzer(String pathFragment, LinkedList<Predicate> filterList) {
+ this.pathFragment = pathFragment;
+ this.filterList = filterList;
+ }
+
+ static PathToken analyze(String pathFragment, LinkedList<Predicate> filterList) {
+ return new PathComponentAnalyzer(pathFragment, filterList).analyze();
+ }
+
+ public PathToken analyze() {
+
+ if ("$".equals(pathFragment)) return new RootPathToken();
+ else if ("..".equals(pathFragment)) return new ScanPathToken();
+ else if ("[*]".equals(pathFragment)) return new WildcardPathToken();
+ else if (".*".equals(pathFragment)) return new WildcardPathToken();
+ else if ("[?]".equals(pathFragment)) return new PredicatePathToken(filterList.poll());
+
+ else if (FILTER_PATTERN.matcher(pathFragment).matches()) {
+ final int criteriaCount = Utils.countMatches(pathFragment, "?");
+ List<Predicate> filters = new ArrayList<>(criteriaCount);
+ for (int i = 0; i < criteriaCount; i++) {
+ filters.add(filterList.poll());
+ }
+ return new PredicatePathToken(filters);
+ }
+
+ this.i = 0;
+ do {
+ current = pathFragment.charAt(i);
+
+ switch (current) {
+ case '?':
+ return analyzeCriteriaSequence4();
+ case '\'':
+ return analyzeProperty();
+ default:
+ if (Character.isDigit(current) || current == ':' || current == '-' || current == '@') {
+ return analyzeArraySequence();
+ }
+ i++;
+ break;
+ }
+
+
+ } while (i < pathFragment.length());
+
+ //"Could not analyze path component: " + pathFragment
+ return null;
+ }
+
+
+ public PathToken analyzeCriteriaSequence4() {
+ int[] bounds = findFilterBounds();
+ if (bounds == null) {
+ return null;
+ }
+ i = bounds[1];
+
+ return new PredicatePathToken(Filter.parse(pathFragment.substring(bounds[0], bounds[1])));
+ }
+
+ int[] findFilterBounds() {
+ int end = 0;
+ int start = i;
+
+ while (pathFragment.charAt(start) != '[') {
+ start--;
+ }
+
+ int mem = ' ';
+ int curr = start;
+ boolean inProp = false;
+ int openSquareBracket = 0;
+ int openBrackets = 0;
+ while (end == 0) {
+ char c = pathFragment.charAt(curr);
+ switch (c) {
+ case '(':
+ if (!inProp) openBrackets++;
+ break;
+ case ')':
+ if (!inProp) openBrackets--;
+ break;
+ case '[':
+ if (!inProp) openSquareBracket++;
+ break;
+ case ']':
+ if (!inProp) {
+ openSquareBracket--;
+ if (openBrackets == 0) {
+ end = curr + 1;
+ }
+ }
+ break;
+ case '\'':
+ if (mem == '\\') {
+ break;
+ }
+ inProp = !inProp;
+ break;
+ default:
+ break;
+ }
+ mem = c;
+ curr++;
+ }
+ if (openBrackets != 0 || openSquareBracket != 0) {
+ // "Filter brackets are not balanced"
+ return null;
+ }
+ return new int[]{start, end};
+ }
+
+
+ //"['foo']"
+ private PathToken analyzeProperty() {
+ List<String> properties = new ArrayList<String>();
+ StringBuilder buffer = new StringBuilder();
+
+ boolean propertyIsOpen = false;
+
+ while (current != ']') {
+ switch (current) {
+ case '\'':
+ if (propertyIsOpen) {
+ properties.add(buffer.toString());
+ buffer.setLength(0);
+ propertyIsOpen = false;
+ } else {
+ propertyIsOpen = true;
+ }
+ break;
+ default:
+ if (propertyIsOpen) {
+ buffer.append(current);
+ }
+ break;
+ }
+ current = pathFragment.charAt(++i);
+ }
+ return new PropertyPathToken(properties);
+ }
+
+
+ //"[-1:]" sliceFrom
+ //"[:1]" sliceTo
+ //"[0:5]" sliceBetween
+ //"[1]"
+ //"[1,2,3]"
+ //"[(@.length - 1)]"
+ private PathToken analyzeArraySequence() {
+ StringBuilder buffer = new StringBuilder();
+ List<Integer> numbers = new ArrayList<Integer>();
+
+ boolean contextSize = (current == '@');
+ boolean sliceTo = false;
+ boolean sliceFrom = false;
+ boolean sliceBetween = false;
+ boolean indexSequence = false;
+ boolean singleIndex = false;
+
+ if (contextSize) {
+
+ current = pathFragment.charAt(++i);
+ current = pathFragment.charAt(++i);
+ while (current != '-') {
+ if (current == ' ' || current == '(' || current == ')') {
+ current = pathFragment.charAt(++i);
+ continue;
+ }
+ buffer.append(current);
+ current = pathFragment.charAt(++i);
+ }
+ String function = buffer.toString();
+ buffer.setLength(0);
+ if (!function.equals("size") && !function.equals("length")) {
+ // "Invalid function: @." + function + ". Supported functions are: [(@.length - n)] and [(@.size() - n)]"
+ return null;
+ }
+ while (current != ')') {
+ if (current == ' ') {
+ current = pathFragment.charAt(++i);
+ continue;
+ }
+ buffer.append(current);
+ current = pathFragment.charAt(++i);
+ }
+
+ } else {
+
+
+ while (Character.isDigit(current) || current == ',' || current == ' ' || current == ':' || current == '-') {
+
+ switch (current) {
+ case ' ':
+ break;
+ case ':':
+ if (buffer.length() == 0) {
+ //this is a tail slice [:12]
+ sliceTo = true;
+ current = pathFragment.charAt(++i);
+ while (Character.isDigit(current) || current == ' ' || current == '-') {
+ if (current != ' ') {
+ buffer.append(current);
+ }
+ current = pathFragment.charAt(++i);
+ }
+ numbers.add(Integer.parseInt(buffer.toString()));
+ buffer.setLength(0);
+ } else {
+ //we now this starts with [12:???
+ numbers.add(Integer.parseInt(buffer.toString()));
+ buffer.setLength(0);
+ current = pathFragment.charAt(++i);
+
+ //this is a tail slice [:12]
+ while (Character.isDigit(current) || current == ' ' || current == '-') {
+ if (current != ' ') {
+ buffer.append(current);
+ }
+ current = pathFragment.charAt(++i);
+ }
+
+ if (buffer.length() == 0) {
+ sliceFrom = true;
+ } else {
+ sliceBetween = true;
+ numbers.add(Integer.parseInt(buffer.toString()));
+ buffer.setLength(0);
+ }
+ }
+ break;
+ case ',':
+ numbers.add(Integer.parseInt(buffer.toString()));
+ buffer.setLength(0);
+ indexSequence = true;
+ break;
+ default:
+ buffer.append(current);
+ break;
+ }
+ if (current == ']') {
+ break;
+ }
+ current = pathFragment.charAt(++i);
+ }
+ }
+ if (buffer.length() > 0) {
+ numbers.add(Integer.parseInt(buffer.toString()));
+ }
+ singleIndex = (numbers.size() == 1) && !sliceTo && !sliceFrom && !contextSize;
+
+ ArrayPathToken.Operation operation = null;
+
+ if (singleIndex) operation = ArrayPathToken.Operation.SINGLE_INDEX;
+ else if (indexSequence) operation = ArrayPathToken.Operation.INDEX_SEQUENCE;
+ else if (sliceFrom) operation = ArrayPathToken.Operation.SLICE_FROM;
+ else if (sliceTo) operation = ArrayPathToken.Operation.SLICE_TO;
+ else if (sliceBetween) operation = ArrayPathToken.Operation.SLICE_BETWEEN;
+ else if (contextSize) operation = ArrayPathToken.Operation.CONTEXT_SIZE;
+
+ assert operation != null;
+
+ return new ArrayPathToken(numbers, operation);
+
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4618f46f/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
index c5ff814..058e21c 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
@@ -67,6 +67,18 @@ public class TestEvaluateJsonPath {
Assert.fail("Processor incorrectly ran with an invalid configuration of multiple paths specified as attributes for a destination of content.");
}
+ @Test(expected = AssertionError.class)
+ public void testInvalidConfiguration_invalidJsonPath_space() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
+ testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_CONTENT);
+ testRunner.setProperty("JsonPath1", "$[0]. _id");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ Assert.fail("Processor incorrectly ran with an invalid configuration of multiple paths specified as attributes for a destination of content.");
+ }
+
@Test
public void testConfiguration_destinationAttributes_twoPaths() throws Exception {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
[16/50] incubator-nifi git commit: Merge branch 'json-processors' of
https://github.com/apiri/incubator-nifi into NIFI-360
Posted by mc...@apache.org.
Merge branch 'json-processors' of https://github.com/apiri/incubator-nifi into NIFI-360
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/f05cc938
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/f05cc938
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/f05cc938
Branch: refs/heads/NIFI-353
Commit: f05cc9383cc86f3697aed59edefff00d35336838
Parents: dde5fd5 9a5b6d5
Author: joewitt <jo...@apache.org>
Authored: Tue Feb 17 22:17:05 2015 -0500
Committer: joewitt <jo...@apache.org>
Committed: Tue Feb 17 22:17:05 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 28 ++
.../nifi-standard-processors/pom.xml | 4 +
.../processors/standard/EvaluateJsonPath.java | 240 +++++++++++
.../nifi/processors/standard/SplitJson.java | 143 +++++++
.../processors/standard/util/JsonUtils.java | 127 ++++++
.../org.apache.nifi.processor.Processor | 2 +
.../standard/TestEvaluateJsonPath.java | 252 +++++++++++
.../nifi/processors/standard/TestSplitJson.java | 126 ++++++
.../test/resources/TestJson/json-sample.json | 415 +++++++++++++++++++
nifi/pom.xml | 5 +
10 files changed, 1342 insertions(+)
----------------------------------------------------------------------
[37/50] incubator-nifi git commit: NIFI-296: Add tests for ooxml
office documents
Posted by mc...@apache.org.
NIFI-296: Add tests for ooxml office documents
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/8f5afd0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/8f5afd0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/8f5afd0c
Branch: refs/heads/NIFI-353
Commit: 8f5afd0c9af068e8050810ed51ee2a055fcad379
Parents: 089eec2
Author: Adam Lamar <ad...@gmail.com>
Authored: Sun Mar 1 19:49:49 2015 +0000
Committer: Adam Lamar <ad...@gmail.com>
Committed: Sun Mar 1 19:58:54 2015 +0000
----------------------------------------------------------------------
.../processors/standard/TestIdentifyMimeType.java | 8 ++++++++
.../resources/TestIdentifyMimeType/392790.pptx | Bin 0 -> 49539 bytes
.../resources/TestIdentifyMimeType/528206.xlsx | Bin 0 -> 13153 bytes
.../resources/TestIdentifyMimeType/651924.docx | Bin 0 -> 12131 bytes
4 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/8f5afd0c/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
index 1bf4585..97c4b34 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
@@ -72,6 +72,11 @@ public class TestIdentifyMimeType {
expectedMimeTypes.put("1.xml", "application/xml");
expectedMimeTypes.put("flowfilev3", "application/flowfile-v3");
expectedMimeTypes.put("flowfilev1.tar", "application/flowfile-v1");
+ // Office documents below randomly selected from govdocs1:
+ // http://digitalcorpora.org/corpora/govdocs
+ expectedMimeTypes.put("651924.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
+ expectedMimeTypes.put("528206.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ expectedMimeTypes.put("392790.pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation");
final Map<String, String> expectedExtensions = new HashMap<>();
expectedExtensions.put("1.7z", ".7z");
@@ -90,6 +95,9 @@ public class TestIdentifyMimeType {
expectedExtensions.put("1.xml", ".xml");
expectedExtensions.put("flowfilev3", "");
expectedExtensions.put("flowfilev1.tar", "");
+ expectedExtensions.put("651924.docx", ".docx");
+ expectedExtensions.put("528206.xlsx", ".xlsx");
+ expectedExtensions.put("392790.pptx", ".pptx");
final List<MockFlowFile> filesOut = runner.getFlowFilesForRelationship(IdentifyMimeType.REL_SUCCESS);
for (final MockFlowFile file : filesOut) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/8f5afd0c/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx
new file mode 100644
index 0000000..84000b9
Binary files /dev/null and b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/8f5afd0c/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx
new file mode 100644
index 0000000..4ce41fc
Binary files /dev/null and b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/8f5afd0c/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx
new file mode 100644
index 0000000..837dfa1
Binary files /dev/null and b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx differ
[24/50] incubator-nifi git commit: Adding missing subject property on
the JsonPath validator.
Posted by mc...@apache.org.
Adding missing subject property on the JsonPath validator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/caed7f84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/caed7f84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/caed7f84
Branch: refs/heads/NIFI-353
Commit: caed7f8468bfee4db47c892bd17d2e36908fc8d2
Parents: d694860
Author: Aldrin Piri <al...@gmail.com>
Authored: Sat Feb 28 12:36:30 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sat Feb 28 12:36:30 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/processors/standard/AbstractJsonPathProcessor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/caed7f84/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
index 54edc70..70efe38 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -57,7 +57,7 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
} catch (InvalidPathException ipe) {
error = ipe.toString();
}
- return new ValidationResult.Builder().valid(error == null).explanation(error).build();
+ return new ValidationResult.Builder().subject("JsonPath expression " + subject).valid(error == null).explanation(error).build();
}
};
[08/50] incubator-nifi git commit: Adding a test case for SplitJson
where the specified path cannot be found. Adjusting documentation for
additional detail.
Posted by mc...@apache.org.
Adding a test case for SplitJson where the specified path cannot be found. Adjusting documentation for additional detail.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/7a74dd94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/7a74dd94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/7a74dd94
Branch: refs/heads/NIFI-353
Commit: 7a74dd942008bf2178634746e70266f46657d4ef
Parents: 2e05dcb
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 15:50:42 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 15:50:42 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/SplitJson.java | 12 +++++++--
.../processors/standard/util/JsonUtils.java | 28 +++++++++++++++-----
.../nifi/processors/standard/TestSplitJson.java | 11 ++++++++
3 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7a74dd94/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index e589b48..3ca0ace 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -18,6 +18,7 @@ package org.apache.nifi.processors.standard;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
@@ -105,10 +106,17 @@ public class SplitJson extends AbstractProcessor {
final List<FlowFile> segments = new ArrayList<>();
- Object jsonPathResult = documentContext.read(jsonPath);
+ Object jsonPathResult;
+ try {
+ jsonPathResult = documentContext.read(jsonPath);
+ } catch (PathNotFoundException e) {
+ logger.warn("JsonPath {} could not be found for FlowFile {}", new Object[]{jsonPath.getPath(), original});
+ processSession.transfer(original, REL_FAILURE);
+ return;
+ }
if (!(jsonPathResult instanceof List)) {
- logger.error("The evaluated value {} of {} was not an array compatible type and cannot be split.",
+ logger.error("The evaluated value {} of {} was not a JSON Array compatible type and cannot be split.",
new Object[]{jsonPathResult, jsonPath.getPath()});
processSession.transfer(original, REL_FAILURE);
return;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7a74dd94/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
index 0bf33dd..5fe2a56 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -89,14 +89,23 @@ public class JsonUtils {
return contextHolder.get();
}
-
+ /*
+ * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition.
+ * Accordingly, a strict JSON approach is preferred in determining whether or not a document is valid.
+ */
+
+ /**
+ * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition, accordingly a strict JSON approach
+ * is preferred in determining whether or not a document is valid.
+ * Performs a validation of the provided stream according to RFC 4627 as implemented by {@link net.minidev.json.parser.JSONParser#MODE_RFC4627}
+ *
+ * @param inputStream of content to be validated as JSON
+ * @return true, if the content is valid within the bounds of the strictness specified; false otherwise
+ * @throws IOException
+ */
public static boolean isValidJson(InputStream inputStream) throws IOException {
boolean isValid = false;
- /*
- * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition.
- * Accordingly, a strict JSON approach is preferred in determining whether or not a document is valid.
- */
try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) {
isValid = JSONValue.isValidJsonStrict(inputStreamReader);
}
@@ -104,8 +113,15 @@ public class JsonUtils {
return isValid;
}
+ /**
+ * Determines the context by which JsonSmartJsonProvider would treat the value. {@link java.util.Map} and
+ * {@link java.util.List} objects can be rendered as JSON elements, everything else is treated as a scalar.
+ *
+ * @param obj item to be inspected if it is a scalar or a JSON element
+ * @return false, if the object is a supported type; true otherwise
+ */
public static boolean isJsonScalar(Object obj) {
- // For the default provider, a Map or List is able to be handled as a JSON entity
+ // For the default provider, JsonSmartJsonProvider, a Map or List is able to be handled as a JSON entity
return !(obj instanceof Map || obj instanceof List);
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7a74dd94/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
index dd6fc6d..f47467f 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
@@ -112,4 +112,15 @@ public class TestSplitJson {
testRunner.getFlowFilesForRelationship(SplitJson.REL_SPLIT).get(0).assertContentEquals("{\"first\":\"Shaffer\",\"last\":\"Pearson\"}");
}
+ @Test
+ public void testSplit_pathNotFound() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$.nonexistent");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ testRunner.assertTransferCount(SplitJson.REL_FAILURE, 1);
+ testRunner.getFlowFilesForRelationship(SplitJson.REL_FAILURE).get(0).assertContentEquals(JSON_SNIPPET);
+ }
}
[39/50] incubator-nifi git commit: NIFI-389: Fixed bug that caused
services that depend on other services to not be locatable in mock framework
Posted by mc...@apache.org.
NIFI-389: Fixed bug that caused services that depend on other services to not be locatable in mock framework
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/96a42cd7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/96a42cd7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/96a42cd7
Branch: refs/heads/NIFI-353
Commit: 96a42cd72b0f009377080d471c420e14d23eca89
Parents: 83b33c8
Author: Mark Payne <ma...@hotmail.com>
Authored: Sun Mar 1 15:30:42 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Sun Mar 1 15:30:42 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/util/MockControllerServiceLookup.java | 3 +++
.../src/main/java/org/apache/nifi/util/MockProcessContext.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/96a42cd7/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java b/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
index f5a716c..8298a39 100644
--- a/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
+++ b/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
@@ -47,6 +47,9 @@ public abstract class MockControllerServiceLookup implements ControllerServiceLo
controllerServiceMap.remove(service.getIdentifier());
}
+ protected void addControllerServices(final MockControllerServiceLookup other) {
+ this.controllerServiceMap.putAll(other.controllerServiceMap);
+ }
protected ControllerServiceConfiguration getConfiguration(final String identifier) {
return controllerServiceMap.get(identifier);
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/96a42cd7/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java b/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
index f83301c..6536928 100644
--- a/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
+++ b/nifi/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
@@ -67,11 +67,13 @@ public class MockProcessContext extends MockControllerServiceLookup implements S
annotationData = context.getControllerServiceAnnotationData(component);
final Map<PropertyDescriptor, String> props = context.getControllerServiceProperties(component);
properties.putAll(props);
+
+ super.addControllerServices(context);
} catch (IllegalArgumentException e) {
// do nothing...the service is being loaded
}
}
-
+
@Override
public PropertyValue getProperty(final PropertyDescriptor descriptor) {
return getProperty(descriptor.getName());
[03/50] incubator-nifi git commit: Adjusting comments and providing a
test to ensure a non-scalar value for a specified scalar return type is
routed to failure.
Posted by mc...@apache.org.
Adjusting comments and providing a test to ensure a non-scalar value for a specified scalar return type is routed to failure.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/0a19ada0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/0a19ada0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/0a19ada0
Branch: refs/heads/NIFI-353
Commit: 0a19ada0a3229b1c8a72fe638f286db33b6f7ba6
Parents: bcebba6
Author: Aldrin Piri <al...@gmail.com>
Authored: Mon Feb 16 23:26:45 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Mon Feb 16 23:26:45 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 31 ++++++++++----------
.../standard/TestEvaluateJsonPath.java | 19 ++++++++++++
2 files changed, 34 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0a19ada0/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index b82764d..fe89635 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -50,16 +50,17 @@ import java.util.*;
@SideEffectFree
@SupportsBatching
@Tags({"JSON", "evaluate", "JsonPath"})
-@CapabilityDescription("Evaluates one or more JsonPath expressions against the content of a FlowFile. The results of those expressions are assigned to "
- + "FlowFile Attributes or are written to the content of the FlowFile itself, depending on configuration of the "
- + "Processor. JsonPaths are entered by adding user-defined properties; the name of the property maps to the Attribute "
- + "Name into which the result will be placed (if the Destination is flowfile-attribute; otherwise, the property name is ignored). "
- + "The value of the property must be a valid JsonPath expression. If the JsonPath evaluates to a JSON array or JSON object and the Return Type is "
- + "set to 'scalar' the FlowFile will be unmodified and will be routed to failure. If the JsonPath does not "
- + "evaluate to a scalar, the FlowFile will be routed to 'unmatched' without having its contents modified. If Destination is "
- + "flowfile-attribute and the expression matches nothing, attributes will be created with empty strings as the value, and the "
- + "FlowFile will always be routed to 'matched.' If Destination is 'flowfile-content' and the expression matches nothing, "
- + "the FlowFile will be routed to 'unmatched' without having its contents modified.")
+@CapabilityDescription("Evaluates one or more JsonPath expressions against the content of a FlowFile. "
+ + "The results of those expressions are assigned to FlowFile Attributes or are written to the content of the FlowFile itself, "
+ + "depending on configuration of the Processor. "
+ + "JsonPaths are entered by adding user-defined properties; the name of the property maps to the Attribute Name "
+ + "into which the result will be placed (if the Destination is flowfile-attribute; otherwise, the property name is ignored). "
+ + "The value of the property must be a valid JsonPath expression. "
+ + "If the JsonPath evaluates to a JSON array or JSON object and the Return Type is set to 'scalar' the FlowFile will be unmodified and will be routed to failure. "
+ + "A Return Type of JSON can return scalar values if the provided JsonPath evaluates to the specified value and will be routed as a match."
+ + "If Destination is 'flowfile-content' and the JsonPath does not evaluate to a defined path, the FlowFile will be routed to 'unmatched' without having its contents modified. "
+ + "If Destination is flowfile-attribute and the expression matches nothing, attributes will be created with "
+ + "empty strings as the value, and the FlowFile will always be routed to 'matched.'")
public class EvaluateJsonPath extends AbstractProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
@@ -174,6 +175,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
final String destination = processContext.getProperty(DESTINATION).getValue();
+ final String returnType = processContext.getProperty(RETURN_TYPE).getValue();
flowFileLoop:
for (FlowFile flowFile : flowFiles) {
@@ -215,7 +217,6 @@ public class EvaluateJsonPath extends AbstractProcessor {
final Map<String, String> jsonPathResults = new HashMap<>();
- // Iterate through all JsonPath entries specified
jsonPathEvalLoop:
for (final Map.Entry<String, JsonPath> attributeJsonPathEntry : attributeToJsonPathMap.entrySet()) {
@@ -226,8 +227,9 @@ public class EvaluateJsonPath extends AbstractProcessor {
final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
try {
Object result = documentContext.read(jsonPathExp);
- if (RETURN_TYPE.getName().equals(RETURN_TYPE_SCALAR) && !isScalar(result)) {
- logger.error("Unable to return a scalar value for a JsonPath {} for FlowFile {}. Transferring to {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), REL_FAILURE.getName()});
+ if (returnType.equals(RETURN_TYPE_SCALAR) && !isScalar(result)) {
+ logger.error("Unable to return a scalar value for the expression {} for FlowFile {}. Evaluated value was {}. Transferring to {}.",
+ new Object[]{jsonPathExp.getPath(), flowFile.getId(), result.toString(), REL_FAILURE.getName()});
processSession.transfer(flowFile, REL_FAILURE);
continue flowFileLoop;
}
@@ -273,9 +275,6 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
private static boolean isScalar(Object obj) {
- /*
- * A given path could be a JSON object or a single/scalar value
- */
return (obj instanceof String);
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/0a19ada0/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
index e72d069..c873969 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
@@ -228,4 +228,23 @@ public class TestEvaluateJsonPath {
testRunner.getFlowFilesForRelationship(expectedRel).get(0).assertContentEquals(JSON_SNIPPET);
}
+
+ @Test
+ public void testRouteFailure_returnTypeScalar_resultArray() throws Exception {
+ String jsonPathAttrKey = "friends.indefinite.id.list";
+
+ final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
+ testRunner.setProperty(EvaluateJsonPath.RETURN_TYPE, EvaluateJsonPath.RETURN_TYPE_SCALAR);
+ testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_CONTENT);
+ testRunner.setProperty(jsonPathAttrKey, "$[0].friends[?(@.id < 3)].id");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ Relationship expectedRel = EvaluateJsonPath.REL_FAILURE;
+
+ testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
+ testRunner.getFlowFilesForRelationship(expectedRel).get(0).assertContentEquals(JSON_SNIPPET);
+ }
+
}
[11/50] incubator-nifi git commit: Adding license terms for asm
dependency introduced via JsonPath -> Json-Smart
Posted by mc...@apache.org.
Adding license terms for asm dependency introduced via JsonPath -> Json-Smart
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/a058fd04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/a058fd04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/a058fd04
Branch: refs/heads/NIFI-353
Commit: a058fd0460dd5ca91560b1239dd733da37cfa782
Parents: 627bd91
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 16:58:04 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 16:58:04 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/a058fd04/nifi/LICENSE
----------------------------------------------------------------------
diff --git a/nifi/LICENSE b/nifi/LICENSE
index 2e67c09..4d8be01 100644
--- a/nifi/LICENSE
+++ b/nifi/LICENSE
@@ -455,4 +455,32 @@ This product bundles 'json2.js' which is available in the 'public domain'.
This product bundles 'reset.css' which is available in the 'public domain'.
For details see http://meyerweb.com/eric/tools/css/reset/
+This product bundles 'asm' which is available under an MIT style license.
+For details see http://asm.ow2.org/asmdex-license.html
+ Copyright (c) 2012 France Télécom
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
[22/50] incubator-nifi git commit: NIFI-387: If possible don't use
ContentRepository.importFrom but just copy stream directly in
StandardProcessSession
Posted by mc...@apache.org.
NIFI-387: If possible don't use ContentRepository.importFrom but just copy stream directly in StandardProcessSession
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/739f0c25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/739f0c25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/739f0c25
Branch: refs/heads/NIFI-353
Commit: 739f0c25e2c885bcfa6eef81a9e4896e618ec0fb
Parents: 50744bf
Author: Mark Payne <ma...@hotmail.com>
Authored: Thu Feb 26 09:04:05 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Thu Feb 26 09:04:05 2015 -0500
----------------------------------------------------------------------
.../repository/StandardProcessSession.java | 46 +++++++++++---------
1 file changed, 26 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/739f0c25/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index dcb461c..8d2e456 100644
--- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -2351,35 +2351,41 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
public FlowFile importFrom(final InputStream source, final FlowFile destination) {
validateRecordState(destination);
final StandardRepositoryRecord record = records.get(destination);
- final ContentClaim newClaim;
+ ContentClaim newClaim = null;
long claimOffset = 0L;
- final boolean appendToClaim = isMergeContent();
- if (appendToClaim) {
- enforceCurrentWriteClaimState();
- newClaim = currentWriteClaim;
- claimOffset = currentWriteClaimSize;
- } else {
- try {
- newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant());
- claimLog.debug("Creating ContentClaim {} for 'importFrom' for {}", newClaim, destination);
- } catch (final IOException e) {
- throw new FlowFileAccessException("Unable to create ContentClaim due to " + e.toString(), e);
- }
- }
-
final long newSize;
+ final boolean appendToClaim = isMergeContent();
try {
- final boolean append = isMergeContent();
- newSize = context.getContentRepository().importFrom(source, newClaim, append);
- bytesWritten.increment(newSize);
- currentWriteClaimSize += newSize;
+ if (appendToClaim) {
+ enforceCurrentWriteClaimState();
+ newClaim = currentWriteClaim;
+ claimOffset = currentWriteClaimSize;
+
+ final long bytesCopied = StreamUtils.copy(source, currentWriteClaimStream);
+ bytesWritten.increment(bytesCopied);
+ currentWriteClaimSize += bytesCopied;
+ newSize = bytesCopied;
+ } else {
+ try {
+ newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant());
+ claimLog.debug("Creating ContentClaim {} for 'importFrom' for {}", newClaim, destination);
+
+ newSize = context.getContentRepository().importFrom(source, newClaim, appendToClaim);
+ bytesWritten.increment(newSize);
+ currentWriteClaimSize += newSize;
+ } catch (final IOException e) {
+ throw new FlowFileAccessException("Unable to create ContentClaim due to " + e.toString(), e);
+ }
+ }
} catch (final Throwable t) {
if (appendToClaim) {
resetWriteClaims();
}
- destroyContent(newClaim);
+ if ( newClaim != null ) {
+ destroyContent(newClaim);
+ }
throw new FlowFileAccessException("Failed to import data from " + source + " for " + destination + " due to " + t.toString(), t);
}
[31/50] incubator-nifi git commit: Removing JsonUtils as all
functionality was migrated into AbstractJsonPathProcessor given its limited
utility outside of those classes. Adjusting validation approach for JsonPath
processors to accomodate caching of exp
Posted by mc...@apache.org.
Removing JsonUtils as all functionality was migrated into AbstractJsonPathProcessor given its limited utility outside of those classes. Adjusting validation approach for JsonPath processors to accomodate caching of expressions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/4d3cff35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/4d3cff35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/4d3cff35
Branch: refs/heads/NIFI-353
Commit: 4d3cff3592d16d1ce5608b20a025edf34a7c69d7
Parents: b1f9713
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 13:26:03 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 13:26:03 2015 -0500
----------------------------------------------------------------------
.../standard/AbstractJsonPathProcessor.java | 41 +++++++++++------
.../processors/standard/EvaluateJsonPath.java | 43 +++++++++++++++++-
.../nifi/processors/standard/SplitJson.java | 32 +++++++++++--
.../processors/standard/util/JsonUtils.java | 47 --------------------
4 files changed, 98 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d3cff35/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
index 02547f3..baeef7b 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -51,19 +51,6 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
private static final JsonProvider JSON_PROVIDER = STRICT_PROVIDER_CONFIGURATION.jsonProvider();
- public static final Validator JSON_PATH_VALIDATOR = new Validator() {
- @Override
- public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
- String error = null;
- try {
- JsonPath compile = JsonPath.compile(input);
- } catch (InvalidPathException ipe) {
- error = ipe.toString();
- }
- return new ValidationResult.Builder().subject(subject).valid(error == null).explanation(error).build();
- }
- };
-
static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
// Parse the document once into an associated context to support multiple path evaluations if specified
final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
@@ -99,4 +86,32 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
return JSON_PROVIDER.toJson(jsonPathResult);
}
+ protected abstract static class JsonPathValidator implements Validator {
+
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ JsonPath compiledJsonPath = null;
+ String error = null;
+ try {
+ if (isStale(subject, input)) {
+ compiledJsonPath = JsonPath.compile(input);
+ cacheComputedValue(subject, input, compiledJsonPath);
+ }
+ } catch (InvalidPathException ipe) {
+ error = ipe.toString();
+ }
+ return new ValidationResult.Builder().subject(subject).valid(error == null).explanation(error).build();
+ }
+
+ /**
+ * An optional hook to act on the compute value
+ */
+ abstract void cacheComputedValue(String subject, String input, JsonPath computedJsonPath);
+
+ /**
+ * A hook for implementing classes to determine if a cached value is stale for a compiled JsonPath represented
+ * by either a validation
+ */
+ abstract boolean isStale(String subject, String input);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d3cff35/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 1b89dee..b40f6c6 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -20,11 +20,13 @@ import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.InvalidJsonException;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnRemoved;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
@@ -38,12 +40,14 @@ import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.util.ObjectHolder;
-import org.apache.nifi.util.StringUtils;
+import org.apache.nifi.util.Tuple;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
@EventDriven
@SideEffectFree
@@ -92,6 +96,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
private Set<Relationship> relationships;
private List<PropertyDescriptor> properties;
+ private ConcurrentMap<String, Tuple<String, JsonPath>> cachedJsonPathMap = new ConcurrentHashMap<>();
@Override
protected void init(final ProcessorInitializationContext context) {
@@ -145,13 +150,47 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.expressionLanguageSupported(false)
- .addValidator(JSON_PATH_VALIDATOR)
+ .addValidator(new JsonPathValidator() {
+ @Override
+ public void cacheComputedValue(String subject, String input, JsonPath computedJsonPath) {
+ cachedJsonPathMap.put(subject, new Tuple<>(input, computedJsonPath));
+
+ }
+
+ @Override
+ public boolean isStale(String subject, String input) {
+ return cachedJsonPathMap.get(subject) == null;
+ }
+ })
.required(false)
.dynamic(true)
.build();
}
@Override
+ public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
+ if (descriptor.isDynamic()) {
+ if (!StringUtils.equals(oldValue, newValue)) {
+ cachedJsonPathMap.remove(descriptor.getName());
+ }
+ }
+ }
+
+ /**
+ * Provides cleanup of the map for any JsonPath values that may have been created. This will remove common values
+ * shared between multiple instances, but will be regenerated when the next validation cycle occurs as a result of
+ * isStale()
+ */
+ @OnRemoved
+ public void onRemoved() {
+ for (PropertyDescriptor propertyDescriptor : getPropertyDescriptors()) {
+ if (propertyDescriptor.isDynamic()) {
+ cachedJsonPathMap.remove(propertyDescriptor.getName());
+ }
+ }
+ }
+
+ @Override
public void onTrigger(ProcessContext processContext, final ProcessSession processSession) throws ProcessException {
FlowFile flowFile = processSession.get();
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d3cff35/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 5177bdd..8c7ae4d 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -20,6 +20,7 @@ import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.InvalidJsonException;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
@@ -38,6 +39,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
@EventDriven
@SideEffectFree
@@ -53,7 +56,17 @@ public class SplitJson extends AbstractJsonPathProcessor {
.name("JsonPath Expression")
.description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
.required(true)
- .addValidator(JSON_PATH_VALIDATOR)
+ .addValidator(new JsonPathValidator() {
+ @Override
+ public void cacheComputedValue(String subject, String input, JsonPath computedJson) {
+ JSON_PATH_MAP.put(input, computedJson);
+ }
+
+ @Override
+ public boolean isStale(String subject, String input) {
+ return JSON_PATH_MAP.get(input) == null;
+ }
+ })
.build();
public static final Relationship REL_ORIGINAL = new Relationship.Builder().name("original").description("The original FlowFile that was split into segments. If the FlowFile fails processing, nothing will be sent to this relationship").build();
@@ -63,6 +76,8 @@ public class SplitJson extends AbstractJsonPathProcessor {
private List<PropertyDescriptor> properties;
private Set<Relationship> relationships;
+ private static final ConcurrentMap<String, JsonPath> JSON_PATH_MAP = new ConcurrentHashMap();
+
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> properties = new ArrayList<>();
@@ -87,6 +102,16 @@ public class SplitJson extends AbstractJsonPathProcessor {
}
@Override
+ public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
+ if (descriptor.equals(ARRAY_JSON_PATH_EXPRESSION)) {
+ if (!StringUtils.equals(oldValue, newValue)) {
+ // clear the cached item
+ JSON_PATH_MAP.remove(oldValue);
+ }
+ }
+ }
+
+ @Override
public void onTrigger(final ProcessContext processContext, final ProcessSession processSession) {
final FlowFile original = processSession.get();
if (original == null) {
@@ -104,8 +129,9 @@ public class SplitJson extends AbstractJsonPathProcessor {
return;
}
- final String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
- final JsonPath jsonPath = JsonPath.compile(jsonPathExpression);
+ String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
+ final JsonPath jsonPath = JSON_PATH_MAP.get(jsonPathExpression);
+ getLogger().info("Using value {} for split ", new Object[]{jsonPathExpression});
final List<FlowFile> segments = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d3cff35/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
deleted file mode 100644
index 68b18b8..0000000
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.standard.util;
-
-import net.minidev.json.JSONValue;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Provides utilities for interacting with JSON elements
- *
- * @see <a href="http://json.org">http://json.org</a>
- */
-public class JsonUtils {
-
- /**
- * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition, accordingly a strict JSON approach
- * is preferred in determining whether or not a document is valid.
- * Performs a validation of the provided stream according to RFC 4627 as implemented by {@link net.minidev.json.parser.JSONParser#MODE_RFC4627}
- *
- * @param inputStream of content to be validated as JSON
- * @return true, if the content is valid within the bounds of the strictness specified; false otherwise
- * @throws IOException
- */
- public static boolean isValidJson(InputStream inputStream) throws IOException {
- try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) {
- return JSONValue.isValidJsonStrict(inputStreamReader);
- }
- }
-
-}
[35/50] incubator-nifi git commit: NIFI-221: Finished initial
implementation of http procs
Posted by mc...@apache.org.
NIFI-221: Finished initial implementation of http procs
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/7c990541
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/7c990541
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/7c990541
Branch: refs/heads/NIFI-353
Commit: 7c99054183428b0a696cdf16be46267089f9faf4
Parents: c53b0f9
Author: Mark Payne <ma...@hotmail.com>
Authored: Sun Mar 1 14:31:26 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Sun Mar 1 14:31:26 2015 -0500
----------------------------------------------------------------------
.../processors/standard/HandleHttpRequest.java | 66 +++++--
.../standard/TestHandleHttpRequest.java | 127 ++++++++++++++
.../standard/TestHandleHttpResponse.java | 172 +++++++++++++++++++
3 files changed, 353 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7c990541/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index 98b5fdd..e72df7d 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -17,8 +17,10 @@
package org.apache.nifi.processors.standard;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URLDecoder;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Collections;
@@ -73,6 +75,7 @@ import com.sun.jersey.api.client.ClientResponse.Status;
@CapabilityDescription("Starts an HTTP Server and listens for HTTP Requests. For each request, creates a FlowFile and transfers to 'success'. This Processor is designed to be used in conjunction with the HandleHttpResponse Processor in order to create a Web Service")
public class HandleHttpRequest extends AbstractProcessor {
public static final String HTTP_CONTEXT_ID = "http.context.identifier";
+ private static final Pattern URL_QUERY_PARAM_DELIMITER = Pattern.compile("&");
// Allowable values for client auth
public static final AllowableValue CLIENT_NONE = new AllowableValue("No Authentication", "Processor will not authenticate clients. Anyone can communicate with this Processor anonymously");
@@ -107,6 +110,13 @@ public class HandleHttpRequest extends AbstractProcessor {
.required(false)
.identifiesControllerService(SSLContextService.class)
.build();
+ public static final PropertyDescriptor URL_CHARACTER_SET = new PropertyDescriptor.Builder()
+ .name("URL Character Set")
+ .description("The character set to use for decoding URL parameters")
+ .required(true)
+ .defaultValue("UTF-8")
+ .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
+ .build();
public static final PropertyDescriptor PATH_REGEX = new PropertyDescriptor.Builder()
.name("Allowed Paths")
.description("A Regular Expression that specifies the valid HTTP Paths that are allowed in the incoming URL Requests. If this value is specified and the path of the HTTP Requests does not match this Regular Expression, the Processor will respond with a 404: NotFound")
@@ -189,6 +199,7 @@ public class HandleHttpRequest extends AbstractProcessor {
descriptors.add(SSL_CONTEXT);
descriptors.add(HTTP_CONTEXT_MAP);
descriptors.add(PATH_REGEX);
+ descriptors.add(URL_CHARACTER_SET);
descriptors.add(ALLOW_GET);
descriptors.add(ALLOW_POST);
descriptors.add(ALLOW_PUT);
@@ -360,7 +371,7 @@ public class HandleHttpRequest extends AbstractProcessor {
protected int getPort() {
for ( final Connector connector : server.getConnectors() ) {
if ( connector instanceof ServerConnector ) {
- return ((ServerConnector) connector).getPort();
+ return ((ServerConnector) connector).getLocalPort();
}
}
@@ -421,19 +432,50 @@ public class HandleHttpRequest extends AbstractProcessor {
return;
}
+ final String charset = context.getProperty(URL_CHARACTER_SET).getValue();
+
final String contextIdentifier = UUID.randomUUID().toString();
final Map<String, String> attributes = new HashMap<>();
- putAttribute(attributes, HTTP_CONTEXT_ID, contextIdentifier);
- putAttribute(attributes, "mime.type", request.getContentType());
- putAttribute(attributes, "http.servlet.path", request.getServletPath());
- putAttribute(attributes, "http.context.path", request.getContextPath());
- putAttribute(attributes, "http.method", request.getMethod());
- putAttribute(attributes, "http.query.string", request.getQueryString());
- putAttribute(attributes, "http.remote.host", request.getRemoteHost());
- putAttribute(attributes, "http.remote.addr", request.getRemoteAddr());
- putAttribute(attributes, "http.remote.user", request.getRemoteUser());
- putAttribute(attributes, "http.request.uri", request.getRequestURI());
- putAttribute(attributes, "http.auth.type", request.getAuthType());
+ try {
+ putAttribute(attributes, HTTP_CONTEXT_ID, contextIdentifier);
+ putAttribute(attributes, "mime.type", request.getContentType());
+ putAttribute(attributes, "http.servlet.path", request.getServletPath());
+ putAttribute(attributes, "http.context.path", request.getContextPath());
+ putAttribute(attributes, "http.method", request.getMethod());
+ if ( request.getQueryString() != null ) {
+ putAttribute(attributes, "http.query.string", URLDecoder.decode(request.getQueryString(), charset));
+ }
+ putAttribute(attributes, "http.remote.host", request.getRemoteHost());
+ putAttribute(attributes, "http.remote.addr", request.getRemoteAddr());
+ putAttribute(attributes, "http.remote.user", request.getRemoteUser());
+ putAttribute(attributes, "http.request.uri", request.getRequestURI());
+ putAttribute(attributes, "http.auth.type", request.getAuthType());
+
+ final String queryString = request.getQueryString();
+ if ( queryString != null ) {
+ final String[] params = URL_QUERY_PARAM_DELIMITER.split(queryString);
+ for ( final String keyValueString : params ) {
+ final int indexOf = keyValueString.indexOf("=");
+ if ( indexOf < 0 ) {
+ // no =, then it's just a key with no value
+ attributes.put("http.query.param." + URLDecoder.decode(keyValueString, charset), "");
+ } else {
+ final String key = keyValueString.substring(0, indexOf);
+ final String value;
+
+ if ( indexOf == keyValueString.length() - 1 ) {
+ value = "";
+ } else {
+ value = keyValueString.substring(indexOf + 1);
+ }
+
+ attributes.put("http.query.param." + URLDecoder.decode(key, charset), URLDecoder.decode(value, charset));
+ }
+ }
+ }
+ } catch (final UnsupportedEncodingException uee) {
+ throw new ProcessException("Invalid character encoding", uee); // won't happen because charset has been validated
+ }
final Enumeration<String> headerNames = request.getHeaderNames();
while ( headerNames.hasMoreElements() ) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7c990541/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
new file mode 100644
index 0000000..85f35e2
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.servlet.AsyncContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.http.HttpContextMap;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.stream.io.NullOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHandleHttpRequest {
+
+ @Test
+ public void testRequestAddedToService() throws InitializationException, MalformedURLException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(HandleHttpRequest.class);
+ runner.setProperty(HandleHttpRequest.PORT, "0");
+
+ final MockHttpContextMap contextMap = new MockHttpContextMap();
+ runner.addControllerService("http-context-map", contextMap);
+ runner.enableControllerService(contextMap);
+ runner.setProperty(HandleHttpRequest.HTTP_CONTEXT_MAP, "http-context-map");
+
+ // trigger processor to stop but not shutdown.
+ runner.run(1, false);
+ try {
+ final Thread httpThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ final int port = ((HandleHttpRequest) runner.getProcessor()).getPort();
+ final HttpURLConnection connection = (HttpURLConnection) new URL("http://localhost:" + port + "/my/path?query=true&value1=value1&value2=&value3&value4=apple=orange").openConnection();
+ connection.setDoOutput(false);
+ connection.setRequestMethod("GET");
+ connection.setRequestProperty("header1", "value1");
+ connection.setRequestProperty("header2", "");
+ connection.setRequestProperty("header3", "apple=orange");
+ connection.setConnectTimeout(3000);
+ connection.setReadTimeout(3000);
+
+ StreamUtils.copy(connection.getInputStream(), new NullOutputStream());
+ } catch (final Throwable t) {
+ t.printStackTrace();
+ Assert.fail(t.toString());
+ }
+ }
+ });
+ httpThread.start();
+
+ try { Thread.sleep(100L); } catch (final InterruptedException ie) {}
+
+ // process the request.
+ runner.run(1, false);
+
+ runner.assertAllFlowFilesTransferred(HandleHttpRequest.REL_SUCCESS, 1);
+ assertEquals(1, contextMap.size());
+
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(HandleHttpRequest.REL_SUCCESS).get(0);
+ mff.assertAttributeEquals("http.query.param.query", "true");
+ mff.assertAttributeEquals("http.query.param.value1", "value1");
+ mff.assertAttributeEquals("http.query.param.value2", "");
+ mff.assertAttributeEquals("http.query.param.value3", "");
+ mff.assertAttributeEquals("http.query.param.value4", "apple=orange");
+ mff.assertAttributeEquals("http.headers.header1", "value1");
+ mff.assertAttributeEquals("http.headers.header3", "apple=orange");
+ } finally {
+ // shut down the server
+ runner.run(1, true);
+ }
+ }
+
+
+ private static class MockHttpContextMap extends AbstractControllerService implements HttpContextMap {
+ private final ConcurrentMap<String, HttpServletResponse> responseMap = new ConcurrentHashMap<>();
+
+ @Override
+ public boolean register(String identifier, HttpServletRequest request, HttpServletResponse response, AsyncContext context) {
+ responseMap.put(identifier, response);
+ return true;
+ }
+
+ @Override
+ public HttpServletResponse getResponse(String identifier) {
+ return responseMap.get(identifier);
+ }
+
+ @Override
+ public void complete(String identifier) {
+ responseMap.remove(identifier);
+ }
+
+ public int size() {
+ return responseMap.size();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7c990541/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
new file mode 100644
index 0000000..7b41809
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpResponse.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.servlet.AsyncContext;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.WriteListener;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.http.HttpContextMap;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestHandleHttpResponse {
+
+ @Test
+ public void testEnsureCompleted() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(HandleHttpResponse.class);
+
+ final MockHttpContextMap contextMap = new MockHttpContextMap("my-id");
+ runner.addControllerService("http-context-map", contextMap);
+ runner.enableControllerService(contextMap);
+ runner.setProperty(HandleHttpResponse.HTTP_CONTEXT_MAP, "http-context-map");
+ runner.setProperty(HandleHttpResponse.STATUS_CODE, "${status.code}");
+ runner.setProperty("my-attr", "${my-attr}");
+ runner.setProperty("no-valid-attr", "${no-valid-attr}");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put(HandleHttpResponse.HTTP_CONTEXT_ID, "my-id");
+ attributes.put("my-attr", "hello");
+ attributes.put("status.code", "201");
+
+ runner.enqueue("hello".getBytes(), attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(HandleHttpResponse.REL_SUCCESS, 1);
+
+ assertEquals("hello", contextMap.baos.toString());
+ assertEquals("hello", contextMap.headersSent.get("my-attr"));
+ assertNull(contextMap.headersSent.get("no-valid-attr"));
+ assertEquals(201, contextMap.statusCode);
+ assertEquals(1, contextMap.getCompletionCount());
+ assertTrue(contextMap.headersWithNoValue.isEmpty());
+ }
+
+
+ private static class MockHttpContextMap extends AbstractControllerService implements HttpContextMap {
+ private final String id;
+ private final AtomicInteger completedCount = new AtomicInteger(0);
+ private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ private final ConcurrentMap<String, String> headersSent = new ConcurrentHashMap<>();
+ private volatile int statusCode = -1;
+
+ private final List<String> headersWithNoValue = new CopyOnWriteArrayList<>();
+
+ public MockHttpContextMap(final String expectedIdentifier) {
+ this.id = expectedIdentifier;
+ }
+
+ @Override
+ public boolean register(String identifier, HttpServletRequest request, HttpServletResponse response, AsyncContext context) {
+ return true;
+ }
+
+ @Override
+ public HttpServletResponse getResponse(final String identifier) {
+ if ( !id.equals(identifier) ) {
+ Assert.fail("attempting to respond to wrong request; should have been " + id + " but was " + identifier);
+ }
+
+ try {
+ final HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Mockito.when(response.getOutputStream()).thenReturn(new ServletOutputStream() {
+ @Override
+ public boolean isReady() { return true; }
+
+ @Override
+ public void setWriteListener(WriteListener writeListener) {}
+
+ @Override
+ public void write(int b) throws IOException { baos.write(b); }
+
+ @Override
+ public void write(byte[] b) throws IOException { baos.write(b); }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException { baos.write(b, off, len); }
+ });
+
+
+ Mockito.doAnswer(new Answer<Object>() {
+ @Override
+ public Object answer(final InvocationOnMock invocation) throws Throwable {
+ final String key = invocation.getArgumentAt(0, String.class);
+ final String value = invocation.getArgumentAt(1, String.class);
+ if ( value == null ) {
+ headersWithNoValue.add(key);
+ } else {
+ headersSent.put(key, value);
+ }
+
+ return null;
+ }
+ }).when(response).setHeader(Mockito.any(String.class), Mockito.any(String.class));
+
+ Mockito.doAnswer(new Answer<Object>() {
+ @Override
+ public Object answer(final InvocationOnMock invocation) throws Throwable {
+ statusCode = invocation.getArgumentAt(0, int.class);
+ return null;
+ }
+ }).when(response).setStatus(Mockito.anyInt());
+
+ return response;
+ } catch (final Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.toString());
+ return null;
+ }
+ }
+
+ @Override
+ public void complete(final String identifier) {
+ if ( !id.equals(identifier) ) {
+ Assert.fail("attempting to respond to wrong request; should have been " + id + " but was " + identifier);
+ }
+
+ completedCount.incrementAndGet();
+ }
+
+ public int getCompletionCount() {
+ return completedCount.get();
+ }
+ }
+}
[34/50] incubator-nifi git commit: Adjusting onRemoved methods for
both JsonPath processors to clean up entries on exit.
Posted by mc...@apache.org.
Adjusting onRemoved methods for both JsonPath processors to clean up entries on exit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/484687a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/484687a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/484687a6
Branch: refs/heads/NIFI-353
Commit: 484687a67b12fb5fc13dabd7851f83a6e4a898be
Parents: 5a2a8fc
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 14:16:22 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 14:16:22 2015 -0500
----------------------------------------------------------------------
.../processors/standard/AbstractJsonPathProcessor.java | 2 +-
.../nifi/processors/standard/EvaluateJsonPath.java | 2 +-
.../org/apache/nifi/processors/standard/SplitJson.java | 12 ++++++++++++
3 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/484687a6/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
index baeef7b..febc3f8 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -86,7 +86,7 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
return JSON_PROVIDER.toJson(jsonPathResult);
}
- protected abstract static class JsonPathValidator implements Validator {
+ abstract static class JsonPathValidator implements Validator {
@Override
public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/484687a6/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index b40f6c6..81c9bbe 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -182,7 +182,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
* isStale()
*/
@OnRemoved
- public void onRemoved() {
+ public void onRemoved(ProcessContext processContext) {
for (PropertyDescriptor propertyDescriptor : getPropertyDescriptors()) {
if (propertyDescriptor.isDynamic()) {
cachedJsonPathMap.remove(propertyDescriptor.getName());
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/484687a6/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 59f4d71..7bb8c4e 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -26,6 +26,7 @@ import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnRemoved;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
@@ -111,6 +112,17 @@ public class SplitJson extends AbstractJsonPathProcessor {
}
}
+ /**
+ * Provides cleanup of the map for any JsonPath values that may have been created. This will remove common values
+ * shared between multiple instances, but will be regenerated when the next validation cycle occurs as a result of
+ * isStale()
+ */
+ @OnRemoved
+ public void onRemoved(ProcessContext processContext) {
+ String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
+ JSON_PATH_MAP.remove(jsonPathExpression);
+ }
+
@Override
public void onTrigger(final ProcessContext processContext, final ProcessSession processSession) {
final FlowFile original = processSession.get();
[47/50] incubator-nifi git commit: Merge branch
'NIFI-296-extend-IdentifyMimeType' of
https://github.com/adamonduty/incubator-nifi into develop
Posted by mc...@apache.org.
Merge branch 'NIFI-296-extend-IdentifyMimeType' of https://github.com/adamonduty/incubator-nifi into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/ee795a7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/ee795a7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/ee795a7d
Branch: refs/heads/NIFI-353
Commit: ee795a7d01c6ec35cfa8866c80f285040c559c81
Parents: fff3759 8f5afd0
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Mar 2 10:12:28 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Mar 2 10:12:28 2015 -0500
----------------------------------------------------------------------
.../nifi-standard-processors/pom.xml | 5 +
.../processors/standard/IdentifyMimeType.java | 405 ++-----------------
.../org/apache/tika/mime/custom-mimetypes.xml | 83 ++++
.../standard/TestIdentifyMimeType.java | 88 ++--
.../test/resources/TestIdentifyMimeType/1.tar | Bin 2048 -> 10240 bytes
.../resources/TestIdentifyMimeType/1.tar.gz | Bin 0 -> 154 bytes
.../test/resources/TestIdentifyMimeType/1.txt | 1 +
.../resources/TestIdentifyMimeType/392790.pptx | Bin 0 -> 49539 bytes
.../resources/TestIdentifyMimeType/528206.xlsx | Bin 0 -> 13153 bytes
.../resources/TestIdentifyMimeType/651924.docx | Bin 0 -> 12131 bytes
10 files changed, 168 insertions(+), 414 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/ee795a7d/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
[38/50] incubator-nifi git commit: NIFI-221: Added additionally
flowfile attributes
Posted by mc...@apache.org.
NIFI-221: Added additionally flowfile attributes
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/83b33c80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/83b33c80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/83b33c80
Branch: refs/heads/NIFI-353
Commit: 83b33c8050c024404209ad504935a50d478e0db1
Parents: 7c99054
Author: Mark Payne <ma...@hotmail.com>
Authored: Sun Mar 1 15:12:24 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Sun Mar 1 15:12:24 2015 -0500
----------------------------------------------------------------------
.../processors/standard/HandleHttpRequest.java | 48 ++++++++++++++++++--
.../standard/TestHandleHttpRequest.java | 3 +-
2 files changed, 47 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/83b33c80/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index e72df7d..575bf95 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -39,6 +39,7 @@ import java.util.regex.Pattern;
import javax.security.cert.X509Certificate;
import javax.servlet.AsyncContext;
import javax.servlet.ServletException;
+import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -111,8 +112,8 @@ public class HandleHttpRequest extends AbstractProcessor {
.identifiesControllerService(SSLContextService.class)
.build();
public static final PropertyDescriptor URL_CHARACTER_SET = new PropertyDescriptor.Builder()
- .name("URL Character Set")
- .description("The character set to use for decoding URL parameters")
+ .name("Default URL Character Set")
+ .description("The character set to use for decoding URL parameters if the HTTP Request does not supply one")
.required(true)
.defaultValue("UTF-8")
.addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
@@ -432,7 +433,7 @@ public class HandleHttpRequest extends AbstractProcessor {
return;
}
- final String charset = context.getProperty(URL_CHARACTER_SET).getValue();
+ final String charset = request.getCharacterEncoding() == null ? context.getProperty(URL_CHARACTER_SET).getValue() : request.getCharacterEncoding();
final String contextIdentifier = UUID.randomUUID().toString();
final Map<String, String> attributes = new HashMap<>();
@@ -442,6 +443,8 @@ public class HandleHttpRequest extends AbstractProcessor {
putAttribute(attributes, "http.servlet.path", request.getServletPath());
putAttribute(attributes, "http.context.path", request.getContextPath());
putAttribute(attributes, "http.method", request.getMethod());
+ putAttribute(attributes, "http.local.addr", request.getLocalAddr());
+ putAttribute(attributes, "http.local.name", request.getLocalName());
if ( request.getQueryString() != null ) {
putAttribute(attributes, "http.query.string", URLDecoder.decode(request.getQueryString(), charset));
}
@@ -449,8 +452,39 @@ public class HandleHttpRequest extends AbstractProcessor {
putAttribute(attributes, "http.remote.addr", request.getRemoteAddr());
putAttribute(attributes, "http.remote.user", request.getRemoteUser());
putAttribute(attributes, "http.request.uri", request.getRequestURI());
+ putAttribute(attributes, "http.request.url", request.getRequestURL().toString());
putAttribute(attributes, "http.auth.type", request.getAuthType());
+ putAttribute(attributes, "http.requested.session.id", request.getRequestedSessionId());
+ if ( request.getDispatcherType() != null ) {
+ putAttribute(attributes, "http.dispatcher.type", request.getDispatcherType().name());
+ }
+ putAttribute(attributes, "http.character.encoding", request.getCharacterEncoding());
+ putAttribute(attributes, "http.locale", request.getLocale());
+ putAttribute(attributes, "http.server.name", request.getServerName());
+ putAttribute(attributes, "http.server.port", request.getServerPort());
+
+ final Enumeration<String> paramEnumeration = request.getParameterNames();
+ while ( paramEnumeration.hasMoreElements() ) {
+ final String paramName = paramEnumeration.nextElement();
+ final String value = request.getParameter(paramName);
+ attributes.put("http.param." + paramName, value);
+ }
+
+ final Cookie[] cookies = request.getCookies();
+ if ( cookies != null ) {
+ for ( final Cookie cookie : cookies ) {
+ final String name = cookie.getName();
+ final String cookiePrefix = "http.cookie." + name + ".";
+ attributes.put(cookiePrefix + "value", cookie.getValue());
+ attributes.put(cookiePrefix + "domain", cookie.getDomain());
+ attributes.put(cookiePrefix + "path", cookie.getPath());
+ attributes.put(cookiePrefix + "max.age", String.valueOf(cookie.getMaxAge()));
+ attributes.put(cookiePrefix + "version", String.valueOf(cookie.getVersion()));
+ attributes.put(cookiePrefix + "secure", String.valueOf(cookie.getSecure()));
+ }
+ }
+
final String queryString = request.getQueryString();
if ( queryString != null ) {
final String[] params = URL_QUERY_PARAM_DELIMITER.split(queryString);
@@ -527,6 +561,14 @@ public class HandleHttpRequest extends AbstractProcessor {
getLogger().info("Transferring {} to 'success'; received from {}", new Object[] {flowFile, request.getRemoteAddr()});
}
+ private void putAttribute(final Map<String, String> map, final String key, final Object value) {
+ if ( value == null ) {
+ return;
+ }
+
+ putAttribute(map, key, value.toString());
+ }
+
private void putAttribute(final Map<String, String> map, final String key, final String value) {
if ( value == null ) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/83b33c80/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
index 85f35e2..363fc0f 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestHandleHttpRequest.java
@@ -78,7 +78,8 @@ public class TestHandleHttpRequest {
});
httpThread.start();
- try { Thread.sleep(100L); } catch (final InterruptedException ie) {}
+ // give processor a bit to handle the http request
+ try { Thread.sleep(1000L); } catch (final InterruptedException ie) {}
// process the request.
runner.run(1, false);
[13/50] incubator-nifi git commit: Adding missing apostrophe for
description of SplitJson
Posted by mc...@apache.org.
Adding missing apostrophe for description of SplitJson
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/5f647aa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/5f647aa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/5f647aa4
Branch: refs/heads/NIFI-353
Commit: 5f647aa46523ee98c8f8c7339216f852e422f9c7
Parents: 408f636
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 18:40:12 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 18:40:12 2015 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/standard/SplitJson.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5f647aa4/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 3ca0ace..c2b520c 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -41,7 +41,7 @@ import java.util.*;
@SupportsBatching
@Tags({"json", "split", "jsonpath"})
@CapabilityDescription("Splits a JSON File into multiple, separate FlowFiles for an array element specified by a JsonPath expression. "
- + "Each generated FlowFile is comprised of an element of the specified array and transferred to relationship 'split, "
+ + "Each generated FlowFile is comprised of an element of the specified array and transferred to relationship 'split,' "
+ "with the original file transferred to the 'original' relationship. If the specified JsonPath is not found or "
+ "does not evaluate to an array element, the original file is routed to 'failure' and no files are generated.")
public class SplitJson extends AbstractProcessor {
[48/50] incubator-nifi git commit: NIFI-296: Removed test files for
which licenses were not readily available
Posted by mc...@apache.org.
NIFI-296: Removed test files for which licenses were not readily available
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/5e0026c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/5e0026c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/5e0026c1
Branch: refs/heads/NIFI-353
Commit: 5e0026c14c1fea10a1bc13fc159c9b6671835fa5
Parents: ee795a7
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Mar 2 11:29:01 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Mar 2 11:29:01 2015 -0500
----------------------------------------------------------------------
.../processors/standard/TestIdentifyMimeType.java | 8 --------
.../resources/TestIdentifyMimeType/392790.pptx | Bin 49539 -> 0 bytes
.../resources/TestIdentifyMimeType/528206.xlsx | Bin 13153 -> 0 bytes
.../resources/TestIdentifyMimeType/651924.docx | Bin 12131 -> 0 bytes
4 files changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5e0026c1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
index 97c4b34..1bf4585 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
@@ -72,11 +72,6 @@ public class TestIdentifyMimeType {
expectedMimeTypes.put("1.xml", "application/xml");
expectedMimeTypes.put("flowfilev3", "application/flowfile-v3");
expectedMimeTypes.put("flowfilev1.tar", "application/flowfile-v1");
- // Office documents below randomly selected from govdocs1:
- // http://digitalcorpora.org/corpora/govdocs
- expectedMimeTypes.put("651924.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
- expectedMimeTypes.put("528206.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
- expectedMimeTypes.put("392790.pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation");
final Map<String, String> expectedExtensions = new HashMap<>();
expectedExtensions.put("1.7z", ".7z");
@@ -95,9 +90,6 @@ public class TestIdentifyMimeType {
expectedExtensions.put("1.xml", ".xml");
expectedExtensions.put("flowfilev3", "");
expectedExtensions.put("flowfilev1.tar", "");
- expectedExtensions.put("651924.docx", ".docx");
- expectedExtensions.put("528206.xlsx", ".xlsx");
- expectedExtensions.put("392790.pptx", ".pptx");
final List<MockFlowFile> filesOut = runner.getFlowFilesForRelationship(IdentifyMimeType.REL_SUCCESS);
for (final MockFlowFile file : filesOut) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5e0026c1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx
deleted file mode 100644
index 84000b9..0000000
Binary files a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/392790.pptx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5e0026c1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx
deleted file mode 100644
index 4ce41fc..0000000
Binary files a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/528206.xlsx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/5e0026c1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx
deleted file mode 100644
index 837dfa1..0000000
Binary files a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/651924.docx and /dev/null differ
[30/50] incubator-nifi git commit: Adding processor documentation for
EvaluateJsonPath and SplitJson
Posted by mc...@apache.org.
Adding processor documentation for EvaluateJsonPath and SplitJson
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/b1f97133
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/b1f97133
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/b1f97133
Branch: refs/heads/NIFI-353
Commit: b1f971335a464f8de34d4f360327f1c9e5e02bbe
Parents: 162f02b
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 00:13:22 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 00:13:22 2015 -0500
----------------------------------------------------------------------
.../index.html | 150 +++++++++++++++++++
.../index.html | 79 ++++++++++
2 files changed, 229 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b1f97133/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
new file mode 100644
index 0000000..055a794
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.EvaluateJsonPath/index.html
@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8"/>
+ <title>EvaluateJsonPath</title>
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css"/>
+</head>
+
+<body>
+<!-- Processor Documentation ================================================== -->
+<h2>Description:</h2>
+
+<p>
+ Evaluates one or more JsonPath expressions against the content of a FlowFile. The results of those expressions are
+ assigned to FlowFile Attributes or are written to the content of the FlowFile itself, depending on configuration of
+ the Processor. JsonPaths are entered by adding user-defined properties; the name of the property maps to the
+ Attribute Name into which the result will be placed (if the Destination is flowfile-attribute; otherwise, the
+ property name is ignored). The value of the property must be a valid JsonPath expression. If the JsonPath evaluates
+ to a JSON array or JSON object and the Return Type is set to 'scalar' the FlowFile will be unmodified and will be
+ routed to failure. A Return Type of JSON can return scalar values if the provided JsonPath evaluates to the
+ specified value and will be routed as a match. If Destination is 'flowfile-content' and the JsonPath does not
+ evaluate to a defined path, the FlowFile will be routed to 'unmatched' without having its contents modified. If
+ Destination is flowfile-attribute and the expression matches nothing, attributes will be created with empty
+ strings as the value, and the FlowFile will always be routed to 'matched.'
+
+</p>
+
+<p>
+ <strong>Properties:</strong>
+</p>
+
+<p>
+ In the list below, the names of required properties appear in bold.
+ Any other properties (not in bold) are considered optional. If a
+ property has a default value, it is indicated. If a property
+ supports the use of the NiFi Expression Language (or simply,
+ "expression language"), that is also indicated.
+</p>
+
+<p>
+ <strong>Modifies Attributes:</strong>
+</p>
+
+<p>
+ This processor adds user-defined attributes if the <Destination> property is set to
+ <code>flowfile-attribute</code>.
+</p>
+
+
+<ul>
+ <li>
+ <strong>Destination</strong>
+ <ul>
+ <li>Indicates whether the results of the JsonPath evaluation are
+ written to the FlowFile content or a FlowFile attribute; if using
+ attribute, the attribute's name must be specified in the value of
+ the Attribute Name property.
+ </li>
+ <li>
+ Valid values are:
+ <ul>
+ <li>flowfile-content</li>
+ <li>flowfile-attribute</li>
+ </ul>
+ </li>
+ <li>Default value: flowfile-content</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li>
+ <strong>Return Type</strong>
+ <ul>
+ <li>Indicates the desired return type of the Xpath expressions.
+ Selecting 'auto-detect' will set the return type to 'json' for a
+ Destination of 'flowfile-content', and 'scalar' for a Destination of
+ 'flowfile-attribute'.")
+ </li>
+ <li>
+ Valid values are:
+ <ul>
+ <li>auto-detect</li>
+ <li>json</li>
+ <li>scalar</li>
+ </ul>
+ </li>
+ <li>Default value: auto-detect</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li>
+ user-defined properties
+ <ul>
+ <li>The name of the attribute to put the JsonPath result into if
+ flowfile-attribute is used as the value for the Destination
+ property; if using flowfile-content as the value for the
+ Destination property, this value is ignored.
+ </li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+</ul>
+
+<p>
+ <strong>Relationships:</strong>
+</p>
+<ul>
+ <li>
+ failure
+ <ul>
+ <li>If the JsonPath cannot be evaluated against the content of the
+ FlowFile, then the FlowFile follows this relationship. For
+ example, if the FlowFile does not contain valid JSON.
+ </li>
+ </ul>
+ </li>
+ <li>
+ matched
+ <ul>
+ <li>If the JsonPath is successfully evaluated and the FlowFile is
+ modified as a result, then the FlowFile follows this
+ relationship.
+ </li>
+ </ul>
+ </li>
+ <li>
+ unmatched
+ <ul>
+ <li>If the JsonPath does not match the content of the FlowFile, then
+ the FlowFile follows this relationship.
+ </li>
+ </ul>
+ </li>
+</ul>
+</p>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b1f97133/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
new file mode 100644
index 0000000..6fc8e61
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.SplitJson/index.html
@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8"/>
+ <title>SplitJson</title>
+
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css"/>
+</head>
+
+<body>
+<!-- Processor Documentation ================================================== -->
+<h2>Description:</h2>
+
+<p>
+ This processor splits a JSON File into multiple, separate FlowFiles for an array element specified by a JsonPath
+ expression.
+ Each generated FlowFile is comprised of an element of the specified array and transferred to relationship 'split,'
+ with the original file transferred to the 'original' relationship. If the specified JsonPath is not found or
+ does not evaluate to an array element, the original file is routed to 'failure' and no files are generated.
+</p>
+
+<strong>Properties:</strong>
+</p>
+
+<p>
+ In the list below, the names of required properties appear in bold. Any other properties (not in bold) are
+ considered optional. If a property has a default value, it is indicated. If a property supports the use of the NiFi
+ Expression Language (or simply, "expression language"), that is also indicated.
+</p>
+<ul>
+ <li><strong>JsonPath Expression</strong>
+ <ul>
+ <li>A JsonPath expression that indicates the array element to split into JSON/scalar fragments.</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+</ul>
+
+<p>
+ <strong>Relationships:</strong>
+</p>
+<ul>
+ <li>failure
+ <ul>
+ <li>If a FlowFile fails processing for any reason (for example, the FlowFile is not valid JSON or the
+ specified path does not exist) and does not go down the original relationship.
+ </li>
+ </ul>
+ </li>
+ <li>original
+ <ul>
+ <li>If FlowFiles are successfully split, a copy of the original FlowFile follows this relationship.</li>
+ </ul>
+ </li>
+ <li>split
+ <ul>
+ <li>If FlowFiles are successfully split into one or more files, those split files follow this
+ relationship.
+ </li>
+ </ul>
+ </li>
+</ul>
+
+</body>
+</html>
[05/50] incubator-nifi git commit: Refactoring common JSON/JsonPath
functionality into JsonUtils
Posted by mc...@apache.org.
Refactoring common JSON/JsonPath functionality into JsonUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/59ad1948
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/59ad1948
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/59ad1948
Branch: refs/heads/NIFI-353
Commit: 59ad1948519fa1f8acccfb6eafbda04684edbc96
Parents: 5a81f19
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 10:26:18 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 10:26:18 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 52 ++--------
.../processors/standard/util/JsonPathUtils.java | 44 --------
.../processors/standard/util/JsonUtils.java | 104 +++++++++++++++++++
3 files changed, 111 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/59ad1948/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 05d31da..0bdd20c 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -16,12 +16,9 @@
*/
package org.apache.nifi.processors.standard;
-import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
-import com.jayway.jsonpath.spi.json.JsonProvider;
-import net.minidev.json.JSONValue;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
@@ -34,17 +31,12 @@ import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.*;
import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
-import org.apache.nifi.processors.standard.util.JsonPathUtils;
-import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.processors.standard.util.JsonUtils;
import org.apache.nifi.stream.io.BufferedOutputStream;
-import org.apache.nifi.util.BooleanHolder;
import org.apache.nifi.util.ObjectHolder;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
@@ -95,8 +87,6 @@ public class EvaluateJsonPath extends AbstractProcessor {
private Set<Relationship> relationships;
private List<PropertyDescriptor> properties;
- private static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
-
@Override
protected void init(final ProcessorInitializationContext context) {
@@ -150,7 +140,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.expressionLanguageSupported(false)
- .addValidator(JsonPathUtils.JSON_PATH_VALIDATOR)
+ .addValidator(JsonUtils.JSON_PATH_VALIDATOR)
.required(false)
.dynamic(true)
.build();
@@ -182,42 +172,15 @@ public class EvaluateJsonPath extends AbstractProcessor {
flowFileLoop:
for (FlowFile flowFile : flowFiles) {
- // Validate the JSON document before attempting processing
- final BooleanHolder validJsonHolder = new BooleanHolder(false);
- processSession.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- try (InputStreamReader inputStreamReader = new InputStreamReader(in)) {
- /*
- * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition.
- * Accordingly, a strict JSON approach is preferred in determining whether or not a document is valid.
- */
- boolean validJson = JSONValue.isValidJsonStrict(inputStreamReader);
- validJsonHolder.set(validJson);
- }
- }
- });
- if (!validJsonHolder.get()) {
- logger.error("FlowFile {} did not have valid JSON content.", new Object[]{flowFile.getId()});
+ final DocumentContext documentContext = JsonUtils.validateAndEstablishJsonContext(processSession, flowFile);
+
+ if (documentContext == null) {
+ logger.error("FlowFile {} did not have valid JSON content.", new Object[]{flowFile});
processSession.transfer(flowFile, REL_FAILURE);
continue flowFileLoop;
}
- // Parse the document once into an associated context to support multiple path evaluations if specified
- final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
- processSession.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) {
- DocumentContext ctx = JsonPath.parse(in);
- contextHolder.set(ctx);
- }
- }
- });
-
- final DocumentContext documentContext = contextHolder.get();
-
final Map<String, String> jsonPathResults = new HashMap<>();
jsonPathEvalLoop:
@@ -226,7 +189,6 @@ public class EvaluateJsonPath extends AbstractProcessor {
String jsonPathAttrKey = attributeJsonPathEntry.getKey();
JsonPath jsonPathExp = attributeJsonPathEntry.getValue();
-
final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
try {
Object result = documentContext.read(jsonPathExp);
@@ -274,7 +236,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
if (isScalar(jsonPathResult)) {
return jsonPathResult.toString();
}
- return JSON_PROVIDER.toJson(jsonPathResult);
+ return JsonUtils.JSON_PROVIDER.toJson(jsonPathResult);
}
private static boolean isScalar(Object obj) {
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/59ad1948/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java
deleted file mode 100644
index ecd2ad0..0000000
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonPathUtils.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.standard.util;
-
-import com.jayway.jsonpath.InvalidPathException;
-import com.jayway.jsonpath.JsonPath;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.Validator;
-
-/**
- * Provides utilities for interacting with JsonPath expressions and results
- *
- * @see <a href="https://github.com/jayway/JsonPath">https://github.com/jayway/JsonPath</a>
- */
-public class JsonPathUtils {
-
- public static final Validator JSON_PATH_VALIDATOR = new Validator() {
- @Override
- public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
- String error = null;
- try {
- JsonPath compile = JsonPath.compile(input);
- } catch (InvalidPathException ipe) {
- error = ipe.toString();
- }
- return new ValidationResult.Builder().valid(error == null).explanation(error).build();
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/59ad1948/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
new file mode 100644
index 0000000..6eb567e
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard.util;
+
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.InvalidPathException;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.spi.json.JsonProvider;
+import net.minidev.json.JSONValue;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.util.BooleanHolder;
+import org.apache.nifi.util.ObjectHolder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Provides utilities for interacting with JSON elements and JsonPath expressions and results
+ *
+ * @see <a href="http://json.org">http://json.org</a>
+ * @see <a href="https://github.com/jayway/JsonPath">https://github.com/jayway/JsonPath</a>
+ */
+public class JsonUtils {
+
+ public static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
+
+ public static final Validator JSON_PATH_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ String error = null;
+ try {
+ JsonPath compile = JsonPath.compile(input);
+ } catch (InvalidPathException ipe) {
+ error = ipe.toString();
+ }
+ return new ValidationResult.Builder().valid(error == null).explanation(error).build();
+ }
+ };
+
+ public static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
+
+ final BooleanHolder validJsonHolder = new BooleanHolder(false);
+ processSession.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(InputStream in) throws IOException {
+ validJsonHolder.set(JsonUtils.isValidJson(in));
+ }
+ });
+
+ // Parse the document once into an associated context to support multiple path evaluations if specified
+ final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
+
+ if (validJsonHolder.get()) {
+ processSession.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(InputStream in) throws IOException {
+ try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) {
+ DocumentContext ctx = JsonPath.parse(in);
+ contextHolder.set(ctx);
+ }
+ }
+ });
+ }
+
+ return contextHolder.get();
+ }
+
+
+ public static boolean isValidJson(InputStream inputStream) throws IOException {
+ boolean isValid = false;
+
+ /*
+ * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition.
+ * Accordingly, a strict JSON approach is preferred in determining whether or not a document is valid.
+ */
+ try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) {
+ isValid = JSONValue.isValidJsonStrict(inputStreamReader);
+ }
+
+ return isValid;
+ }
+}
[19/50] incubator-nifi git commit: Removing the batched get of
flowfiles to utilize the framework provided batching support
Posted by mc...@apache.org.
Removing the batched get of flowfiles to utilize the framework provided batching support
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/81234f3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/81234f3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/81234f3a
Branch: refs/heads/NIFI-353
Commit: 81234f3a6dd1effd4c66bcbdbe8d119843b7fdf2
Parents: 2862771
Author: Aldrin Piri <al...@gmail.com>
Authored: Fri Feb 20 16:05:16 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Fri Feb 20 16:05:16 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 103 +++++++++----------
1 file changed, 49 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/81234f3a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index d79a6de..65266ff 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -151,8 +151,8 @@ public class EvaluateJsonPath extends AbstractProcessor {
@Override
public void onTrigger(ProcessContext processContext, final ProcessSession processSession) throws ProcessException {
- List<FlowFile> flowFiles = processSession.get(50);
- if (flowFiles.isEmpty()) {
+ FlowFile flowFile = processSession.get();
+ if (flowFile == null) {
return;
}
@@ -175,66 +175,61 @@ public class EvaluateJsonPath extends AbstractProcessor {
returnType = destination.equals(DESTINATION_CONTENT) ? RETURN_TYPE_JSON : RETURN_TYPE_SCALAR;
}
- flowFileLoop:
- for (FlowFile flowFile : flowFiles) {
+ final DocumentContext documentContext = JsonUtils.validateAndEstablishJsonContext(processSession, flowFile);
- final DocumentContext documentContext = JsonUtils.validateAndEstablishJsonContext(processSession, flowFile);
+ if (documentContext == null) {
+ logger.error("FlowFile {} did not have valid JSON content.", new Object[]{flowFile});
+ processSession.transfer(flowFile, REL_FAILURE);
+ return;
+ }
- if (documentContext == null) {
- logger.error("FlowFile {} did not have valid JSON content.", new Object[]{flowFile});
- processSession.transfer(flowFile, REL_FAILURE);
- continue flowFileLoop;
- }
+ final Map<String, String> jsonPathResults = new HashMap<>();
+
+ jsonPathEvalLoop:
+ for (final Map.Entry<String, JsonPath> attributeJsonPathEntry : attributeToJsonPathMap.entrySet()) {
- final Map<String, String> jsonPathResults = new HashMap<>();
-
- jsonPathEvalLoop:
- for (final Map.Entry<String, JsonPath> attributeJsonPathEntry : attributeToJsonPathMap.entrySet()) {
-
- String jsonPathAttrKey = attributeJsonPathEntry.getKey();
- JsonPath jsonPathExp = attributeJsonPathEntry.getValue();
-
- final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
- try {
- Object result = documentContext.read(jsonPathExp);
- if (returnType.equals(RETURN_TYPE_SCALAR) && !JsonUtils.isJsonScalar(result)) {
- logger.error("Unable to return a scalar value for the expression {} for FlowFile {}. Evaluated value was {}. Transferring to {}.",
- new Object[]{jsonPathExp.getPath(), flowFile.getId(), result.toString(), REL_FAILURE.getName()});
- processSession.transfer(flowFile, REL_FAILURE);
- continue flowFileLoop;
- }
- resultHolder.set(result);
- } catch (PathNotFoundException e) {
- logger.warn("FlowFile {} could not find path {} for attribute key {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), jsonPathAttrKey}, e);
- if (destination.equals(DESTINATION_ATTRIBUTE)) {
- jsonPathResults.put(jsonPathAttrKey, StringUtils.EMPTY);
- continue jsonPathEvalLoop;
- } else {
- processSession.transfer(flowFile, REL_NO_MATCH);
- continue flowFileLoop;
- }
+ String jsonPathAttrKey = attributeJsonPathEntry.getKey();
+ JsonPath jsonPathExp = attributeJsonPathEntry.getValue();
+
+ final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
+ try {
+ Object result = documentContext.read(jsonPathExp);
+ if (returnType.equals(RETURN_TYPE_SCALAR) && !JsonUtils.isJsonScalar(result)) {
+ logger.error("Unable to return a scalar value for the expression {} for FlowFile {}. Evaluated value was {}. Transferring to {}.",
+ new Object[]{jsonPathExp.getPath(), flowFile.getId(), result.toString(), REL_FAILURE.getName()});
+ processSession.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+ resultHolder.set(result);
+ } catch (PathNotFoundException e) {
+ logger.warn("FlowFile {} could not find path {} for attribute key {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), jsonPathAttrKey}, e);
+ if (destination.equals(DESTINATION_ATTRIBUTE)) {
+ jsonPathResults.put(jsonPathAttrKey, StringUtils.EMPTY);
+ continue jsonPathEvalLoop;
+ } else {
+ processSession.transfer(flowFile, REL_NO_MATCH);
+ return;
}
+ }
- final String resultRepresentation = JsonUtils.getResultRepresentation(resultHolder.get());
- switch (destination) {
- case DESTINATION_ATTRIBUTE:
- jsonPathResults.put(jsonPathAttrKey, resultRepresentation);
- break;
- case DESTINATION_CONTENT:
- flowFile = processSession.write(flowFile, new OutputStreamCallback() {
- @Override
- public void process(final OutputStream out) throws IOException {
- try (OutputStream outputStream = new BufferedOutputStream(out)) {
- outputStream.write(resultRepresentation.getBytes(StandardCharsets.UTF_8));
- }
+ final String resultRepresentation = JsonUtils.getResultRepresentation(resultHolder.get());
+ switch (destination) {
+ case DESTINATION_ATTRIBUTE:
+ jsonPathResults.put(jsonPathAttrKey, resultRepresentation);
+ case DESTINATION_CONTENT:
+ flowFile = processSession.write(flowFile, new OutputStreamCallback() {
+ @Override
+ public void process(final OutputStream out) throws IOException {
+ try (OutputStream outputStream = new BufferedOutputStream(out)) {
+ outputStream.write(resultRepresentation.getBytes(StandardCharsets.UTF_8));
}
- });
- break;
- }
+ }
+ });
+ break;
}
- flowFile = processSession.putAllAttributes(flowFile, jsonPathResults);
- processSession.transfer(flowFile, REL_MATCH);
}
+ flowFile = processSession.putAllAttributes(flowFile, jsonPathResults);
+ processSession.transfer(flowFile, REL_MATCH);
}
}
[42/50] incubator-nifi git commit: Adding licensing and notice
information for JsonPath
Posted by mc...@apache.org.
Adding licensing and notice information for JsonPath
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/e6ebaa4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/e6ebaa4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/e6ebaa4c
Branch: refs/heads/NIFI-353
Commit: e6ebaa4ced457785f33df39c8635056d8a683ac4
Parents: 4618f46
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 21:42:56 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 21:42:56 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 22 ++++++++++++++++++++
nifi/nifi-assembly/NOTICE | 4 ++++
.../src/main/resources/META-INF/NOTICE | 6 +++++-
3 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e6ebaa4c/nifi/LICENSE
----------------------------------------------------------------------
diff --git a/nifi/LICENSE b/nifi/LICENSE
index 50150e3..d00d450 100644
--- a/nifi/LICENSE
+++ b/nifi/LICENSE
@@ -455,3 +455,25 @@ This product bundles 'json2.js' which is available in the 'public domain'.
This product bundles 'reset.css' which is available in the 'public domain'.
For details see http://meyerweb.com/eric/tools/css/reset/
+This product bundles source from 'JsonPath'. Specifically the 'PathCompiler'.
+The source is available under an Apache Software License 2.0.
+
+ Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e6ebaa4c/nifi/nifi-assembly/NOTICE
----------------------------------------------------------------------
diff --git a/nifi/nifi-assembly/NOTICE b/nifi/nifi-assembly/NOTICE
index 1667e9e..025d265 100644
--- a/nifi/nifi-assembly/NOTICE
+++ b/nifi/nifi-assembly/NOTICE
@@ -466,6 +466,10 @@ The following binary components are provided under the Apache Software License v
The following NOTICE information applies:
Copyright 2011 JSON-SMART authors
+ (ASLv2) JsonPath
+ The following NOTICE information applies:
+ Copyright 2011 JsonPath authors
+
************************
Common Development and Distribution License 1.1
************************
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/e6ebaa4c/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
index d63a74d..18961c5 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
@@ -82,7 +82,11 @@ The following binary components are provided under the Apache Software License v
(ASLv2) JSON-SMART
The following NOTICE information applies:
Copyright 2011 JSON-SMART authors
-
+
+ (ASLv2) JsonPath
+ The following NOTICE information applies:
+ Copyright 2011 JsonPath authors
+
************************
Common Development and Distribution License 1.1
************************
[45/50] incubator-nifi git commit: NIFI-360 This closes #31
Posted by mc...@apache.org.
NIFI-360 This closes #31
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/099865a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/099865a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/099865a1
Branch: refs/heads/NIFI-353
Commit: 099865a1b6dcdccb0253d765738bed140d6724cb
Parents: 67a4faf
Author: joewitt <jo...@apache.org>
Authored: Sun Mar 1 23:12:50 2015 -0500
Committer: joewitt <jo...@apache.org>
Committed: Sun Mar 1 23:12:50 2015 -0500
----------------------------------------------------------------------
nifi/NOTICE | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/099865a1/nifi/NOTICE
----------------------------------------------------------------------
diff --git a/nifi/NOTICE b/nifi/NOTICE
index ba32a1b..43af4d1 100644
--- a/nifi/NOTICE
+++ b/nifi/NOTICE
@@ -7,5 +7,5 @@ The Apache Software Foundation (http://www.apache.org/).
The source release of this product bundles a modified
version of PathCompiler from https://github.com/jayway/JsonPath
The following notice information applies
- Copyright 2011 Kalle Stenflo, Jochen Berger
+ Copyright 2011 Kalle Stenflo, Jochen Berger
[09/50] incubator-nifi git commit: Removing unnessecary local variable
Posted by mc...@apache.org.
Removing unnessecary local variable
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/1a746212
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/1a746212
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/1a746212
Branch: refs/heads/NIFI-353
Commit: 1a746212bf2e13680d71dd12dfb60df3a20784ab
Parents: 7a74dd9
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 16:04:21 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 16:04:21 2015 -0500
----------------------------------------------------------------------
.../org/apache/nifi/processors/standard/util/JsonUtils.java | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/1a746212/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
index 5fe2a56..efe89ab 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -104,13 +104,9 @@ public class JsonUtils {
* @throws IOException
*/
public static boolean isValidJson(InputStream inputStream) throws IOException {
- boolean isValid = false;
-
try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) {
- isValid = JSONValue.isValidJsonStrict(inputStreamReader);
+ return JSONValue.isValidJsonStrict(inputStreamReader);
}
-
- return isValid;
}
/**
[28/50] incubator-nifi git commit: Providing provenance fork event
for the created segments generated by SplitJson.
Posted by mc...@apache.org.
Providing provenance fork event for the created segments generated by SplitJson.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/57aa5dd6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/57aa5dd6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/57aa5dd6
Branch: refs/heads/NIFI-353
Commit: 57aa5dd63f25c281ba95869bfbfab465465b29f8
Parents: 6a89745
Author: Aldrin Piri <al...@gmail.com>
Authored: Sat Feb 28 16:04:43 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sat Feb 28 16:04:43 2015 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/standard/SplitJson.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/57aa5dd6/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 8df2de0..fe2216a 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -138,6 +138,8 @@ public class SplitJson extends AbstractJsonPathProcessor {
segments.add(split);
}
+ processSession.getProvenanceReporter().fork(original, segments);
+
processSession.transfer(segments, REL_SPLIT);
processSession.transfer(original, REL_ORIGINAL);
logger.info("Split {} into {} FlowFiles", new Object[]{original, segments.size()});
[17/50] incubator-nifi git commit: NIFI-360 providing review feedback
on github PR
Posted by mc...@apache.org.
NIFI-360 providing review feedback on github PR
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/28627712
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/28627712
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/28627712
Branch: refs/heads/NIFI-353
Commit: 2862771235eb7a9e007c1a14096098d9809987be
Parents: f05cc93
Author: joewitt <jo...@apache.org>
Authored: Tue Feb 17 23:29:16 2015 -0500
Committer: joewitt <jo...@apache.org>
Committed: Tue Feb 17 23:29:16 2015 -0500
----------------------------------------------------------------------
nifi/LICENSE | 29 --------------------
nifi/nifi-assembly/LICENSE | 28 +++++++++++++++++++
nifi/nifi-assembly/NOTICE | 4 +++
.../src/main/resources/META-INF/LICENSE | 29 ++++++++++++++++++++
.../src/main/resources/META-INF/NOTICE | 4 +++
5 files changed, 65 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/28627712/nifi/LICENSE
----------------------------------------------------------------------
diff --git a/nifi/LICENSE b/nifi/LICENSE
index 4d8be01..50150e3 100644
--- a/nifi/LICENSE
+++ b/nifi/LICENSE
@@ -455,32 +455,3 @@ This product bundles 'json2.js' which is available in the 'public domain'.
This product bundles 'reset.css' which is available in the 'public domain'.
For details see http://meyerweb.com/eric/tools/css/reset/
-This product bundles 'asm' which is available under an MIT style license.
-For details see http://asm.ow2.org/asmdex-license.html
-
- Copyright (c) 2012 France Télécom
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
- 1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 3. Neither the name of the copyright holders nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/28627712/nifi/nifi-assembly/LICENSE
----------------------------------------------------------------------
diff --git a/nifi/nifi-assembly/LICENSE b/nifi/nifi-assembly/LICENSE
index da81c41..bf9e649 100644
--- a/nifi/nifi-assembly/LICENSE
+++ b/nifi/nifi-assembly/LICENSE
@@ -799,5 +799,33 @@ This product bundles 'JCraft Jzlib' which is available under a 3-Clause BSD Lice
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+This product bundles 'asm' which is available under an MIT style license.
+For details see http://asm.ow2.org/asmdex-license.html
+ Copyright (c) 2012 France Télécom
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/28627712/nifi/nifi-assembly/NOTICE
----------------------------------------------------------------------
diff --git a/nifi/nifi-assembly/NOTICE b/nifi/nifi-assembly/NOTICE
index 7d42538..1667e9e 100644
--- a/nifi/nifi-assembly/NOTICE
+++ b/nifi/nifi-assembly/NOTICE
@@ -462,6 +462,10 @@ The following binary components are provided under the Apache Software License v
This product includes software developed by the
Visigoth Software Society (http://www.visigoths.org/).
+ (ASLv2) JSON-SMART
+ The following NOTICE information applies:
+ Copyright 2011 JSON-SMART authors
+
************************
Common Development and Distribution License 1.1
************************
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/28627712/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE
index 44bb0c0..36ba155 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE
@@ -290,3 +290,32 @@ This product bundles 'JCraft Jzlib' which is available under a 3-Clause BSD Lice
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+This product bundles 'asm' which is available under an MIT style license.
+For details see http://asm.ow2.org/asmdex-license.html
+
+ Copyright (c) 2012 France Télécom
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/28627712/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
index c5a1533..d63a74d 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE
@@ -78,6 +78,10 @@ The following binary components are provided under the Apache Software License v
Written by Doug Lea with assistance from members of JCP JSR-166
Expert Group and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/
+
+ (ASLv2) JSON-SMART
+ The following NOTICE information applies:
+ Copyright 2011 JSON-SMART authors
************************
Common Development and Distribution License 1.1
[29/50] incubator-nifi git commit: Removing the separate reads for
validation preferring to do the read once and handle any exceptions.
Posted by mc...@apache.org.
Removing the separate reads for validation preferring to do the read once and handle any exceptions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/162f02b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/162f02b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/162f02b1
Branch: refs/heads/NIFI-353
Commit: 162f02b12fd7b6d1a78efab09942ceffea49e07d
Parents: 57aa5dd
Author: Aldrin Piri <al...@gmail.com>
Authored: Sat Feb 28 21:24:25 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sat Feb 28 21:24:25 2015 -0500
----------------------------------------------------------------------
.../standard/AbstractJsonPathProcessor.java | 36 ++++++++------------
.../processors/standard/EvaluateJsonPath.java | 15 ++++----
.../nifi/processors/standard/SplitJson.java | 9 ++---
3 files changed, 26 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/162f02b1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
index 70efe38..02547f3 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -20,7 +20,9 @@ import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.InvalidPathException;
import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.internal.spi.json.JsonSmartJsonProvider;
import com.jayway.jsonpath.spi.json.JsonProvider;
+import net.minidev.json.parser.JSONParser;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
@@ -28,9 +30,7 @@ import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.io.InputStreamCallback;
-import org.apache.nifi.processors.standard.util.JsonUtils;
import org.apache.nifi.stream.io.BufferedInputStream;
-import org.apache.nifi.util.BooleanHolder;
import org.apache.nifi.util.ObjectHolder;
import java.io.IOException;
@@ -46,7 +46,10 @@ import java.util.Map;
*/
public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
- protected static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
+ private static final Configuration STRICT_PROVIDER_CONFIGURATION =
+ Configuration.builder().jsonProvider(new JsonSmartJsonProvider(JSONParser.MODE_RFC4627)).build();
+
+ private static final JsonProvider JSON_PROVIDER = STRICT_PROVIDER_CONFIGURATION.jsonProvider();
public static final Validator JSON_PATH_VALIDATOR = new Validator() {
@Override
@@ -57,35 +60,23 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
} catch (InvalidPathException ipe) {
error = ipe.toString();
}
- return new ValidationResult.Builder().subject("JsonPath expression " + subject).valid(error == null).explanation(error).build();
+ return new ValidationResult.Builder().subject(subject).valid(error == null).explanation(error).build();
}
};
static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
-
- final BooleanHolder validJsonHolder = new BooleanHolder(false);
+ // Parse the document once into an associated context to support multiple path evaluations if specified
+ final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
processSession.read(flowFile, new InputStreamCallback() {
@Override
public void process(InputStream in) throws IOException {
- validJsonHolder.set(JsonUtils.isValidJson(in));
+ try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) {
+ DocumentContext ctx = JsonPath.using(STRICT_PROVIDER_CONFIGURATION).parse(bufferedInputStream);
+ contextHolder.set(ctx);
+ }
}
});
- // Parse the document once into an associated context to support multiple path evaluations if specified
- final ObjectHolder<DocumentContext> contextHolder = new ObjectHolder<>(null);
-
- if (validJsonHolder.get()) {
- processSession.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) {
- DocumentContext ctx = JsonPath.parse(in);
- contextHolder.set(ctx);
- }
- }
- });
- }
-
return contextHolder.get();
}
@@ -107,4 +98,5 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
}
return JSON_PROVIDER.toJson(jsonPathResult);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/162f02b1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index ed0f07c..1b89dee 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -17,6 +17,7 @@
package org.apache.nifi.processors.standard;
import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.InvalidJsonException;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import org.apache.nifi.annotation.behavior.EventDriven;
@@ -177,9 +178,10 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
returnType = destination.equals(DESTINATION_CONTENT) ? RETURN_TYPE_JSON : RETURN_TYPE_SCALAR;
}
- final DocumentContext documentContext = validateAndEstablishJsonContext(processSession, flowFile);
-
- if (documentContext == null) {
+ DocumentContext documentContext = null;
+ try {
+ documentContext = validateAndEstablishJsonContext(processSession, flowFile);
+ } catch (InvalidJsonException e) {
logger.error("FlowFile {} did not have valid JSON content.", new Object[]{flowFile});
processSession.transfer(flowFile, REL_FAILURE);
return;
@@ -187,7 +189,6 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
final Map<String, String> jsonPathResults = new HashMap<>();
- jsonPathEvalLoop:
for (final Map.Entry<String, JsonPath> attributeJsonPathEntry : attributeToJsonPathMap.entrySet()) {
String jsonPathAttrKey = attributeJsonPathEntry.getKey();
@@ -207,7 +208,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
logger.warn("FlowFile {} could not find path {} for attribute key {}.", new Object[]{flowFile.getId(), jsonPathExp.getPath(), jsonPathAttrKey}, e);
if (destination.equals(DESTINATION_ATTRIBUTE)) {
jsonPathResults.put(jsonPathAttrKey, StringUtils.EMPTY);
- continue jsonPathEvalLoop;
+ continue;
} else {
processSession.transfer(flowFile, REL_NO_MATCH);
return;
@@ -235,6 +236,4 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
flowFile = processSession.putAllAttributes(flowFile, jsonPathResults);
processSession.transfer(flowFile, REL_MATCH);
}
-
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/162f02b1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index fe2216a..5177bdd 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -17,6 +17,7 @@
package org.apache.nifi.processors.standard;
import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.InvalidJsonException;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import org.apache.nifi.annotation.behavior.EventDriven;
@@ -94,10 +95,10 @@ public class SplitJson extends AbstractJsonPathProcessor {
final ProcessorLog logger = getLogger();
-
- final DocumentContext documentContext = validateAndEstablishJsonContext(processSession, original);
-
- if (documentContext == null) {
+ DocumentContext documentContext = null;
+ try {
+ documentContext = validateAndEstablishJsonContext(processSession, original);
+ } catch (InvalidJsonException e) {
logger.error("FlowFile {} did not have valid JSON content.", new Object[]{original});
processSession.transfer(original, REL_FAILURE);
return;
[15/50] incubator-nifi git commit: Encapsulating the rendering of a
JsonPath result within JsonUtils
Posted by mc...@apache.org.
Encapsulating the rendering of a JsonPath result within JsonUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/9a5b6d5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/9a5b6d5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/9a5b6d5b
Branch: refs/heads/NIFI-353
Commit: 9a5b6d5ba234c471df8faefb0ebf9470a68bebb5
Parents: c88b427
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 21:29:40 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 21:29:40 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/processors/standard/EvaluateJsonPath.java | 10 +---------
.../org/apache/nifi/processors/standard/SplitJson.java | 7 +------
.../apache/nifi/processors/standard/util/JsonUtils.java | 10 +++++++++-
3 files changed, 11 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/9a5b6d5b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 3a64a34..d79a6de 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -215,7 +215,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
}
- final String resultRepresentation = getResultRepresentation(resultHolder.get());
+ final String resultRepresentation = JsonUtils.getResultRepresentation(resultHolder.get());
switch (destination) {
case DESTINATION_ATTRIBUTE:
jsonPathResults.put(jsonPathAttrKey, resultRepresentation);
@@ -237,12 +237,4 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
}
- private static String getResultRepresentation(Object jsonPathResult) {
- if (JsonUtils.isJsonScalar(jsonPathResult)) {
- return jsonPathResult.toString();
- }
- return JsonUtils.JSON_PROVIDER.toJson(jsonPathResult);
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/9a5b6d5b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index c2b520c..78e1b2a 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -129,12 +129,7 @@ public class SplitJson extends AbstractProcessor {
split = processSession.write(split, new OutputStreamCallback() {
@Override
public void process(OutputStream out) throws IOException {
- String resultSegmentContent;
- if (JsonUtils.isJsonScalar(resultSegment)) {
- resultSegmentContent = resultSegment.toString();
- } else {
- resultSegmentContent = JsonUtils.JSON_PROVIDER.toJson(resultSegment);
- }
+ String resultSegmentContent = JsonUtils.getResultRepresentation(resultSegment);
out.write(resultSegmentContent.getBytes(StandardCharsets.UTF_8));
}
});
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/9a5b6d5b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
index 6f97207..2174c1e 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -46,7 +46,7 @@ import java.util.Map;
*/
public class JsonUtils {
- public static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
+ static final JsonProvider JSON_PROVIDER = Configuration.defaultConfiguration().jsonProvider();
public static final Validator JSON_PATH_VALIDATOR = new Validator() {
@Override
@@ -116,4 +116,12 @@ public class JsonUtils {
return !(obj instanceof Map || obj instanceof List);
}
+
+ public static String getResultRepresentation(Object jsonPathResult) {
+ if (JsonUtils.isJsonScalar(jsonPathResult)) {
+ return jsonPathResult.toString();
+ }
+ return JSON_PROVIDER.toJson(jsonPathResult);
+ }
+
}
[27/50] incubator-nifi git commit: Adding a provenance event for
EvaluateJsonPath when content is overwritten with selected expression.
Posted by mc...@apache.org.
Adding a provenance event for EvaluateJsonPath when content is overwritten with selected expression.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/6a89745e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/6a89745e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/6a89745e
Branch: refs/heads/NIFI-353
Commit: 6a89745ec8646acfdacbf27511cb992c821c51b9
Parents: caed7f8
Author: Aldrin Piri <al...@gmail.com>
Authored: Sat Feb 28 15:54:52 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sat Feb 28 15:54:52 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/processors/standard/EvaluateJsonPath.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6a89745e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index f7caa68..ed0f07c 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -227,6 +227,8 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
}
}
});
+ processSession.getProvenanceReporter().modifyContent(flowFile,
+ "Replaced content with result of expression " + jsonPathExp.getPath());
break;
}
}
[26/50] incubator-nifi git commit: NIFI-377: Add EncodeContent
processor
Posted by mc...@apache.org.
NIFI-377: Add EncodeContent processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/b2935410
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/b2935410
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/b2935410
Branch: refs/heads/NIFI-353
Commit: b2935410a2bcae73848e817578f2e777df517a45
Parents: 50744bf
Author: Adam Lamar <ad...@gmail.com>
Authored: Fri Feb 20 15:50:46 2015 +0000
Committer: Adam Lamar <ad...@gmail.com>
Committed: Sat Feb 28 20:28:37 2015 +0000
----------------------------------------------------------------------
.../nifi/processors/standard/EncodeContent.java | 248 +++++++++++++++++++
.../util/ValidatingBase32InputStream.java | 78 ++++++
.../processors/standard/TestEncodeContent.java | 164 ++++++++++++
3 files changed, 490 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b2935410/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
new file mode 100644
index 0000000..2465b56
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Base32InputStream;
+import org.apache.commons.codec.binary.Base32OutputStream;
+
+import org.apache.commons.codec.binary.Base64InputStream;
+import org.apache.commons.codec.binary.Base64OutputStream;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.logging.ProcessorLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processors.standard.util.ValidatingBase32InputStream;
+import org.apache.nifi.processors.standard.util.ValidatingBase64InputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.util.StopWatch;
+
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@Tags({"experimental", "encode", "decode", "base64", "hex"})
+@CapabilityDescription("Encodes the FlowFile content in base64")
+public class EncodeContent extends AbstractProcessor {
+
+ public static final String ENCODE_MODE = "Encode";
+ public static final String DECODE_MODE = "Decode";
+
+ // List of support encodings.
+ public static final String BASE64_ENCODING = "base64";
+ public static final String BASE32_ENCODING = "base32";
+ public static final String HEX_ENCODING = "hex";
+
+ public static final PropertyDescriptor MODE = new PropertyDescriptor.Builder()
+ .name("Mode")
+ .description("Specifies whether the content should be encoded or decoded")
+ .required(true)
+ .allowableValues(ENCODE_MODE, DECODE_MODE)
+ .defaultValue(ENCODE_MODE)
+ .build();
+
+ public static final PropertyDescriptor ENCODING = new PropertyDescriptor.Builder()
+ .name("Encoding")
+ .description("Specifies the type of encoding used")
+ .required(true)
+ .allowableValues(BASE64_ENCODING, BASE32_ENCODING, HEX_ENCODING)
+ .defaultValue(BASE64_ENCODING)
+ .build();
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success").description("Any FlowFile that is successfully encoded or decoded will be routed to success").build();
+ public static final Relationship REL_FAILURE = new Relationship.Builder().name("failure").description("Any FlowFile that cannot be encoded or decoded will be routed to failure").build();
+
+ private List<PropertyDescriptor> properties;
+ private Set<Relationship> relationships;
+
+ @Override
+ protected void init(final ProcessorInitializationContext context) {
+ final List<PropertyDescriptor> props = new ArrayList<>();
+ props.add(MODE);
+ props.add(ENCODING);
+ this.properties = Collections.unmodifiableList(props);
+
+ final Set<Relationship> rels = new HashSet<>();
+ rels.add(REL_SUCCESS);
+ rels.add(REL_FAILURE);
+ this.relationships = Collections.unmodifiableSet(rels);
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ final ProcessorLog logger = getLogger();
+
+ boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
+ String encoding = context.getProperty(ENCODING).getValue();
+ StreamCallback encoder = null;
+
+ // Select the encoder/decoder to use
+ if (encode) {
+ if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
+ encoder = new EncodeBase64();
+ } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
+ encoder = new EncodeBase32();
+ } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
+ encoder = new EncodeHex();
+ }
+ } else {
+ if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
+ encoder = new DecodeBase64();
+ } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
+ encoder = new DecodeBase32();
+ } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
+ encoder = new DecodeHex();
+ }
+ }
+
+ if (encoder == null) {
+ logger.warn("Unknown operation: {} {}", new Object[]{encode ? "encode" : "decode", encoding});
+ return;
+ }
+
+ try {
+ final StopWatch stopWatch = new StopWatch(true);
+ flowFile = session.write(flowFile, encoder);
+
+ logger.info("Successfully {} {}", new Object[]{encode ? "encoded" : "decoded", flowFile});
+ session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
+ session.transfer(flowFile, REL_SUCCESS);
+ } catch (Exception e) {
+ logger.error("Failed to {} {} due to {}", new Object[]{encode ? "encode" : "decode", flowFile, e});
+ session.transfer(flowFile, REL_FAILURE);
+ }
+ }
+
+ private class EncodeBase64 implements StreamCallback {
+ @Override
+ public void process(InputStream in, OutputStream out) throws IOException {
+ try (Base64OutputStream bos = new Base64OutputStream(out)) {
+ StreamUtils.copy(in, bos);
+ }
+ }
+ }
+
+ private class DecodeBase64 implements StreamCallback {
+ @Override
+ public void process(InputStream in, OutputStream out) throws IOException {
+ try (Base64InputStream bis = new Base64InputStream(new ValidatingBase64InputStream(in))) {
+ StreamUtils.copy(bis, out);
+ }
+ }
+ }
+
+ private class EncodeBase32 implements StreamCallback {
+ @Override
+ public void process(InputStream in, OutputStream out) throws IOException {
+ try (Base32OutputStream bos = new Base32OutputStream(out)) {
+ StreamUtils.copy(in, bos);
+ }
+ }
+ }
+
+ private class DecodeBase32 implements StreamCallback {
+ @Override
+ public void process(InputStream in, OutputStream out) throws IOException {
+ try (Base32InputStream bis = new Base32InputStream(new ValidatingBase32InputStream(in))) {
+ StreamUtils.copy(bis, out);
+ }
+ }
+ }
+
+ private static final byte[] HEX_CHARS = {'0', '1', '2', '3', '4', '5', '6', '7',
+ '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
+
+ private class EncodeHex implements StreamCallback {
+ @Override
+ public void process(InputStream in, OutputStream out) throws IOException {
+ int len;
+ byte[] inBuf = new byte[8192];
+ byte[] outBuf = new byte[inBuf.length * 2];
+ while ((len = in.read(inBuf)) > 0) {
+ for (int i = 0; i < len; i++) {
+ outBuf[i*2] = HEX_CHARS[(inBuf[i] & 0xF0) >>> 4];
+ outBuf[i*2 +1] = HEX_CHARS[inBuf[i] & 0x0F];
+ }
+ out.write(outBuf, 0, len*2);
+ }
+ out.flush();
+ }
+ }
+
+ private class DecodeHex implements StreamCallback {
+ @Override
+ public void process(InputStream in, OutputStream out) throws IOException {
+ int len;
+ byte[] inBuf = new byte[8192];
+ Hex h = new Hex();
+ while ((len = in.read(inBuf)) > 0) {
+ // If the input buffer is of odd length, try to get another byte
+ if (len % 2 != 0) {
+ int b = in.read();
+ if (b != -1) {
+ inBuf[len] = (byte) b;
+ len++;
+ }
+ }
+
+ // Construct a new buffer bounded to len
+ byte[] slice = Arrays.copyOfRange(inBuf, 0, len);
+ try {
+ out.write(h.decode(slice));
+ } catch (DecoderException ex) {
+ throw new IOException(ex);
+ }
+ }
+ out.flush();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b2935410/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/ValidatingBase32InputStream.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/ValidatingBase32InputStream.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/ValidatingBase32InputStream.java
new file mode 100644
index 0000000..f7ebeab
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/ValidatingBase32InputStream.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard.util;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import org.apache.commons.codec.binary.Base32;
+
+
+/**
+ * An InputStream that throws an IOException if any byte is read that is not a
+ * valid Base32 character. Whitespace is considered valid.
+ */
+public class ValidatingBase32InputStream extends FilterInputStream {
+
+ private final Base32 b32 = new Base32();
+
+ public ValidatingBase32InputStream(InputStream in) {
+ super(in);
+ }
+
+ @Override
+ public int read(byte[] b, int offset, int len) throws IOException {
+ int numRead = super.read(b, offset, len);
+ if (numRead > 0) {
+ byte[] copy = b;
+ if (numRead < b.length) {
+ // isBase32 checks the whole length of byte[], we need to limit it to numRead
+ copy = Arrays.copyOf(b, numRead);
+ }
+ if (!b32.isInAlphabet(copy, true)) {
+ throw new IOException("Data is not base32 encoded.");
+ }
+ }
+ return numRead;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ int numRead = super.read(b);
+ if (numRead > 0) {
+ byte[] copy = b;
+ if (numRead < b.length) {
+ // isBase32 checks the whole length of byte[], we need to limit it to numRead
+ copy = Arrays.copyOf(b, numRead);
+ }
+ if (!b32.isInAlphabet(copy, true)) {
+ throw new IOException("Data is not base32 encoded.");
+ }
+ }
+ return numRead;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int data = super.read();
+ if (!b32.isInAlphabet((byte) data)) {
+ throw new IOException("Data is not base32 encoded.");
+ }
+ return super.read();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b2935410/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncodeContent.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncodeContent.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncodeContent.java
new file mode 100644
index 0000000..fec411d
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncodeContent.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+
+import org.junit.Test;
+
+public class TestEncodeContent {
+
+ @Test
+ public void testBase64RoundTrip() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.ENCODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.BASE64_ENCODING);
+
+ testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_SUCCESS, 1);
+
+ MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(EncodeContent.REL_SUCCESS).get(0);
+ testRunner.assertQueueEmpty();
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.enqueue(flowFile);
+ testRunner.clearTransferState();
+ testRunner.run();
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_SUCCESS, 1);
+
+ flowFile = testRunner.getFlowFilesForRelationship(EncodeContent.REL_SUCCESS).get(0);
+ flowFile.assertContentEquals(new File("src/test/resources/hello.txt"));
+ }
+
+ @Test
+ public void testFailDecodeNotBase64() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.BASE64_ENCODING);
+
+ testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testFailDecodeNotBase64ButIsAMultipleOfFourBytes() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.BASE64_ENCODING);
+
+ testRunner.enqueue("four@@@@multiple".getBytes());
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testBase32RoundTrip() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.ENCODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.BASE32_ENCODING);
+
+ testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_SUCCESS, 1);
+
+ MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(EncodeContent.REL_SUCCESS).get(0);
+ testRunner.assertQueueEmpty();
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.enqueue(flowFile);
+ testRunner.clearTransferState();
+ testRunner.run();
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_SUCCESS, 1);
+
+ flowFile = testRunner.getFlowFilesForRelationship(EncodeContent.REL_SUCCESS).get(0);
+ flowFile.assertContentEquals(new File("src/test/resources/hello.txt"));
+ }
+
+ @Test
+ public void testFailDecodeNotBase32() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.BASE32_ENCODING);
+
+ testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testHexRoundTrip() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.ENCODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.HEX_ENCODING);
+
+ testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_SUCCESS, 1);
+
+ MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(EncodeContent.REL_SUCCESS).get(0);
+ testRunner.assertQueueEmpty();
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.enqueue(flowFile);
+ testRunner.clearTransferState();
+ testRunner.run();
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_SUCCESS, 1);
+
+ flowFile = testRunner.getFlowFilesForRelationship(EncodeContent.REL_SUCCESS).get(0);
+ flowFile.assertContentEquals(new File("src/test/resources/hello.txt"));
+ }
+
+ @Test
+ public void testFailDecodeNotHex() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new EncodeContent());
+
+ testRunner.setProperty(EncodeContent.MODE, EncodeContent.DECODE_MODE);
+ testRunner.setProperty(EncodeContent.ENCODING, EncodeContent.HEX_ENCODING);
+
+ testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ testRunner.clearTransferState();
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(EncodeContent.REL_FAILURE, 1);
+ }
+}
[14/50] incubator-nifi git commit: Removing extraneous comment block
that was replaced by method Javadoc
Posted by mc...@apache.org.
Removing extraneous comment block that was replaced by method Javadoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/c88b427e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/c88b427e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/c88b427e
Branch: refs/heads/NIFI-353
Commit: c88b427e8201a1e1ba42720def252e0762e31ed9
Parents: 5f647aa
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 18:42:05 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 18:42:05 2015 -0500
----------------------------------------------------------------------
.../org/apache/nifi/processors/standard/util/JsonUtils.java | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c88b427e/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
index efe89ab..6f97207 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -89,11 +89,6 @@ public class JsonUtils {
return contextHolder.get();
}
- /*
- * JSONValue#isValidJson is permissive to the degree of the Smart JSON definition.
- * Accordingly, a strict JSON approach is preferred in determining whether or not a document is valid.
- */
-
/**
* JSONValue#isValidJson is permissive to the degree of the Smart JSON definition, accordingly a strict JSON approach
* is preferred in determining whether or not a document is valid.
[46/50] incubator-nifi git commit: Merge branch
'NIFI-377-EncodeContent' of https://github.com/adamonduty/incubator-nifi into
develop
Posted by mc...@apache.org.
Merge branch 'NIFI-377-EncodeContent' of https://github.com/adamonduty/incubator-nifi into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/fff37591
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/fff37591
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/fff37591
Branch: refs/heads/NIFI-353
Commit: fff37591a3abeb96cede6adf9fed4d5176d8690b
Parents: 099865a b293541
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Mar 2 10:05:29 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Mar 2 10:05:29 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/EncodeContent.java | 248 +++++++++++++++++++
.../util/ValidatingBase32InputStream.java | 78 ++++++
.../processors/standard/TestEncodeContent.java | 164 ++++++++++++
3 files changed, 490 insertions(+)
----------------------------------------------------------------------
[25/50] incubator-nifi git commit: NIFI-221: Fixed shutdown of
HandleHttpRequest and provided documentation
Posted by mc...@apache.org.
NIFI-221: Fixed shutdown of HandleHttpRequest and provided documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/c53b0f9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/c53b0f9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/c53b0f9d
Branch: refs/heads/NIFI-353
Commit: c53b0f9d15390ccee1c8011e6a05db82ad02f74c
Parents: 2d48323
Author: Mark Payne <ma...@hotmail.com>
Authored: Sat Feb 28 14:55:56 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Sat Feb 28 14:55:56 2015 -0500
----------------------------------------------------------------------
.../processors/standard/HandleHttpRequest.java | 1 +
.../index.html | 255 +++++++++++++++++++
.../index.html | 112 ++++++++
3 files changed, 368 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c53b0f9d/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
index 85d7433..98b5fdd 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/HandleHttpRequest.java
@@ -397,6 +397,7 @@ public class HandleHttpRequest extends AbstractProcessor {
if ( server != null ) {
getLogger().debug("Shutting down server");
server.stop();
+ server.destroy();
server.join();
getLogger().info("Shut down {}", new Object[] {server});
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c53b0f9d/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpRequest/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpRequest/index.html b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpRequest/index.html
new file mode 100644
index 0000000..d3da666
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpRequest/index.html
@@ -0,0 +1,255 @@
+<!DOCTYPE html>
+<html lang="en">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <head>
+ <meta charset="utf-8" />
+ <title>HandleHttpRequest</title>
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
+ </head>
+
+ <body>
+ <!-- Processor Documentation ================================================== -->
+ <h2>Description:</h2>
+ <p>
+ This processor starts an HTTP server and creates a FlowFile for each HTTP Request that it receives. The Processor leaves
+ the HTTP Connection open and is intended to be used in conjunction with a
+ <a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a> Processor.
+ </p>
+
+ <p>
+ The pairing of this Processor with a <a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a> Processor
+ provides the ability to use NiFi to visually construct a web server that can carry out any functionality that is available
+ through the existing Processors. For example, one could construct a Web-based front end to an SFTP Server by constructing a
+ flow such as:
+ </p>
+
+ <p>
+ <a href="index.html">HandleHttpRequest</a> ->
+ <a href="../org.apache.nifi.processors.standard.PutSFTP/index.html">PutSFTP</a> ->
+ <a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a>
+ </p>
+
+ <p>
+ The HandleHttpRequest Processor provides several Properties to configure which methods are supported, the paths that are
+ supported, and SSL configuration. The FlowFiles that are generated by this Processor have the following attributes added to
+ them, providing powerful routing capabilities and traceability of all data:
+ </p>
+
+ <table>
+ <thead>
+ <th>Attribute Name</th>
+ <th>Attribute Description</th>
+ </thead>
+ <tbody>
+ <tr>
+ <td>http.context.identifier</td>
+ <td>An identifier that allows the HandleHttpRequest and HandleHttpResponse to coordinate which FlowFile belongs
+ to which HTTP Request/Response.</td>
+ </tr>
+ <tr>
+ <td>mime.type</td>
+ <td>The MIME Type of the data, according to the HTTP Header "Content-Type"</td>
+ </tr>
+ <tr>
+ <td>http.servlet.path</td>
+ <td>The part of the request URL that is considered the Servlet Path</td>
+ </tr>
+ <tr>
+ <td>http.context.path</td>
+ <td>The part of the request URL that is considered to be the Context Path</td>
+ </tr>
+ <tr>
+ <td>http.method</td>
+ <td>The HTTP Method that was used for the request, such as GET or POST</td>
+ </tr>
+ <tr>
+ <td>http.query.string</td>
+ <td>The query string portion of hte Request URL</td>
+ </tr>
+ <tr>
+ <td>http.remote.host</td>
+ <td>The hostname of the requestor</td>
+ </tr>
+ <tr>
+ <td>http.remote.addr</td>
+ <td>The hostname:port combination of the requestor</td>
+ </tr>
+ <tr>
+ <td>http.remote.user</td>
+ <td>The username of the requestor</td>
+ </tr>
+ <tr>
+ <td>http.request.uri</td>
+ <td>The full Request URL</td>
+ </tr>
+ <tr>
+ <td>http.auth.type</td>
+ <td>The type of HTTP Authorization used</td>
+ </tr>
+ <tr>
+ <td>http.principal.name</td>
+ <td>The name of the authenticated user making the request</td>
+ </tr>
+ <tr>
+ <td>http.subject.dn</td>
+ <td>The Distinguished Name of the requestor. This value will not be populated unless the Processor is
+ configured to use an SSLContext Service</td>
+ </tr>
+ <tr>
+ <td>http.issuer.dn</td>
+ <td>The Distinguished Name of the entity that issued the Subject's certificate. This value will not be
+ populated unless the Processor is configured to use an SSLContext Service</td>
+ </tr>
+ <tr>
+ <td>http.headers.XXX</td>
+ <td>Each of the HTTP Headers that is received in the request will be added as an attribute, prefixed
+ with "http.headers." For example, if the request contains an HTTP Header named "x-my-header",
+ then the value will be added to an attribute named "http.headers.x-my-header"</td>
+ </tr>
+ </tbody>
+ </table>
+
+
+ <p>
+ <strong>Properties:</strong>
+ </p>
+ <p>
+ In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. If a property has a default value, it is indicated. If a property supports the use of the NiFi Expression Language (or simply, "expression language"), that is also indicated.
+ </p>
+ <ul>
+ <li><strong>Listening Port</strong>
+ <ul>
+ <li>The port to listen on for incoming HTTP Requests</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li>Hostname
+ <ul>
+ <li>The Hostname to bind to. If not specified, will bind to all hosts</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>HTTP Context Map</strong>
+ <ul>
+ <li>The HTTP Context Map Controller Service to use for caching the HTTP Request Information</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li>SSL Context Service
+ <ul>
+ <li>The Controller Service to use for obtaining an SSL Context. The SSL Context controller service is a mechanism for providing all the security properties that allow for secure communications between NiFi extensions and other systems. See the User Guide or the Controller Services documentation via the "help" link in the upper-right corner of the GUI for more information about the StandardSSLContextService. The value for this property is the identifier name that is configured in the StandardSSLContextService.</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+
+ <li><strong>Allowed Paths</strong>
+ <ul>
+ <li>A Regular Expression that specifies the valid HTTP Paths that are allowed in the incoming URL Requests. If this value is specified and the path of the HTTP Requests does not match this Regular Expression, the Processor will respond with a 404: NotFound</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Allow GET</strong>
+ <ul>
+ <li>Specifies whether or not to allow HTTP GET Method</li>
+ <li>Default value: true</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Allow POST</strong>
+ <ul>
+ <li>Specifies whether or not to allow HTTP POST Method</li>
+ <li>Default value: true</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Allow PUT</strong>
+ <ul>
+ <li>Specifies whether or not to allow HTTP PUT Method</li>
+ <li>Default value: true</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Allow DELETE</strong>
+ <ul>
+ <li>Specifies whether or not to allow HTTP DELETE Method</li>
+ <li>Default value: true</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Allow HEAD</strong>
+ <ul>
+ <li>Specifies whether or not to allow HTTP HEAD Method</li>
+ <li>Default value: false</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Allow OPTIONS</strong>
+ <ul>
+ <li>Specifies whether or not to allow HTTP OPTIONS Method</li>
+ <li>Default value: false</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li>Additional HTTP Methods
+ <ul>
+ <li>A comma-separated list of non-standard HTTP Methods that should be allowed</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ <li><strong>Client Authentication</strong>
+ <ul>
+ <li>Specifies whether or not the Processor should authenticate clients. This value is ignored if the <SSL Context Service> Property is not specified or the SSL Context provided uses only a KeyStore and not a TrustStore.</li>
+ <li>
+ The following values are allowed:
+ <ul>
+ <li><b>No Authentication</b> - Processor will not authenticate clients. Anyone can communicate with this Processor anonymously</li>
+ <li><b>Want Authentication</b> - Processor will try to verify the client but if unable to verify will allow the client to communicate anonymously</li>
+ <li><b>Need Authentication</b> - Processor will reject communications from any client unless the client provides a certificate that is trusted by the TrustStore specified in the SSL Context Service</li>
+ </ul>
+ </li>
+ <li>Default value: No Authentication</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ </ul>
+
+ <p>
+ <strong>Relationships:</strong>
+ </p>
+ <ul>
+ <li>success
+ <ul>
+ <li>All FlowFiles that are created are routed to this relationship.</li>
+ </ul>
+ </li>
+ </ul>
+
+
+ <p>
+ <strong>See Also:</strong><br />
+ <a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a><br />
+ <a href="../org.apache.nifi.http.StandardHttpContextMap/index.html">StandardHttpContextMap</a><br />
+ <a href="../org.apache.nifi.ssl.StandardSSLContextService/index.html">StandardSSLContextService</a><br />
+ </p>
+
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c53b0f9d/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpResponse/index.html
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpResponse/index.html b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpResponse/index.html
new file mode 100644
index 0000000..70d76a6
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.HandleHttpResponse/index.html
@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<html lang="en">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <head>
+ <meta charset="utf-8" />
+ <title>HandleHttpResponse</title>
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
+ </head>
+
+ <body>
+ <!-- Processor Documentation ================================================== -->
+ <h2>Description:</h2>
+ <p>
+ This processor responds to an HTTP request that was received by the
+ <a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a> Processor.
+ </p>
+
+ <p>
+ The pairing of this Processor with a <a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a> Processor
+ provides the ability to use NiFi to visually construct a web server that can carry out any functionality that is available
+ through the existing Processors. For example, one could construct a Web-based front end to an SFTP Server by constructing a
+ flow such as:
+ </p>
+
+ <p>
+ <a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a> ->
+ <a href="../org.apache.nifi.processors.standard.PutSFTP/index.html">PutSFTP</a> ->
+ <a href="index.html">HandleHttpResponse</a>
+ </p>
+
+ <p>
+ This Processor must be configured with the same <HTTP Context Map> service as the corresponding HandleHttpRequest Processor.
+ Otherwise, all FlowFiles will be routed to the 'failure' relationship.
+ </p>
+
+ <p>
+ All FlowFiles must have an attribute named <code>http.context.identifier</code>. The value of this attribute is used to lookup
+ the HTTP Response so that the proper message can be sent back to the requestor. If this attribute is missing, the FlowFile
+ will be routed to 'failure.'
+ </p>
+
+
+ <p>
+ <strong>Properties:</strong>
+ </p>
+ <p>
+ In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. If a property has a default value, it is indicated. If a property supports the use of the NiFi Expression Language (or simply, "expression language"), that is also indicated.
+ </p>
+ <ul>
+ <li><strong>HTTP Status Code</strong>
+ <ul>
+ <li>The HTTP Status Code to use when responding to the HTTP Request. See Section 10 of RFC 2616 for more information.</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: true</li>
+ </ul>
+ </li>
+ <li><strong>HTTP Context Map</strong>
+ <ul>
+ <li>The HTTP Context Map Controller Service to use for caching the HTTP Request Information</li>
+ <li>Default value: no default</li>
+ <li>Supports expression language: false</li>
+ </ul>
+ </li>
+ </ul>
+
+ <p>
+ <strong>Relationships:</strong>
+ </p>
+ <ul>
+ <li>success
+ <ul>
+ <li>If a message is successfully sent back to the requestor, the FlowFile is routed to this relationship.</li>
+ </ul>
+ </li>
+
+ <li>failure
+ <ul>
+ <li>
+ A FlowFile will be routed to failure under the following conditions:
+ <ul>
+ <li>The FlowFile does not have an <code>http.context.identifier</code> attribute</li>
+ <li>The <code>http.context.identifier</code> attribute has a value that cannot be found in the HTTP Context Map</li>
+ <li>The HTTP Status Code is not a number</li>
+ <li>There was a communications failure when attempting to write the response to the requestor</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+ <p>
+ <strong>See Also:</strong><br />
+ <a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a><br />
+ <a href="../org.apache.nifi.http.StandardHttpContextMap/index.html">StandardHttpContextMap</a><br />
+ <a href="../org.apache.nifi.ssl.StandardSSLContextService/index.html">StandardSSLContextService</a><br />
+ </p>
+
+ </body>
+</html>
[21/50] incubator-nifi git commit: Adjusting scope of methods as they
are specific to the JsonPath related processors
Posted by mc...@apache.org.
Adjusting scope of methods as they are specific to the JsonPath related processors
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/d6948601
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/d6948601
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/d6948601
Branch: refs/heads/NIFI-353
Commit: d6948601cd26d5a7c6cfccaaef297cebe91a9d2e
Parents: 46bf048
Author: Aldrin Piri <al...@gmail.com>
Authored: Mon Feb 23 22:51:37 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Mon Feb 23 22:51:37 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/AbstractJsonPathProcessor.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/d6948601/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
index cb4dcf6..54edc70 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractJsonPathProcessor.java
@@ -61,7 +61,7 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
}
};
- public static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
+ static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) {
final BooleanHolder validJsonHolder = new BooleanHolder(false);
processSession.read(flowFile, new InputStreamCallback() {
@@ -101,7 +101,7 @@ public abstract class AbstractJsonPathProcessor extends AbstractProcessor {
return !(obj instanceof Map || obj instanceof List);
}
- public static String getResultRepresentation(Object jsonPathResult) {
+ static String getResultRepresentation(Object jsonPathResult) {
if (isJsonScalar(jsonPathResult)) {
return jsonPathResult.toString();
}
[06/50] incubator-nifi git commit: NIFI-296: Extend capability of
IdentifyMimeType
Posted by mc...@apache.org.
NIFI-296: Extend capability of IdentifyMimeType
This commit backs IdentifyMimeType with the Apache Tika library. Tika
provides detailed mime type identification such as the ability to
differentiate normal zip files from OOXML MS Office documents.
The mime.type attribute continues to be set, though some mime types
have changed due to Tika naming them differently. In addition,
the mime.extension attribute is set to provide the commonly used
extension for the mime type (if known).
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/16fb2b82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/16fb2b82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/16fb2b82
Branch: refs/heads/NIFI-353
Commit: 16fb2b826c0cd983b5d905ceed7aff2a84383d33
Parents: dde5fd5
Author: Adam Lamar <ad...@gmail.com>
Authored: Sat Feb 14 20:57:41 2015 +0000
Committer: Adam Lamar <ad...@gmail.com>
Committed: Tue Feb 17 18:25:17 2015 +0000
----------------------------------------------------------------------
.../nifi-standard-processors/pom.xml | 11 +
.../processors/standard/IdentifyMimeType.java | 426 ++++---------------
.../standard/TestIdentifyMimeType.java | 58 +--
.../test/resources/TestIdentifyMimeType/1.tar | Bin 2048 -> 10240 bytes
.../test/resources/TestIdentifyMimeType/1.txt | 1 +
5 files changed, 97 insertions(+), 399 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/16fb2b82/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index a482e19..b941d03 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -155,5 +155,16 @@
<artifactId>nifi-ssl-context-service</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>1.7</version>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parsers</artifactId>
+ <version>1.7</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/16fb2b82/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
index a52ac3b..c9cb1cd 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
@@ -20,13 +20,14 @@ import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.zip.ZipInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.logging.ProcessorLog;
@@ -44,50 +45,31 @@ import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.util.FlowFilePackagerV1;
import org.apache.nifi.util.FlowFilePackagerV3;
import org.apache.nifi.util.ObjectHolder;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.detect.CompositeDetector;
+import org.apache.tika.detect.DefaultDetector;
+import org.apache.tika.detect.Detector;
+import org.apache.tika.detect.MagicDetector;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeType;
+import org.apache.tika.mime.MimeTypeException;
/**
* <p>
* Attempts to detect the MIME Type of a FlowFile by examining its contents. If
* the MIME Type is determined, it is added to an attribute with the name
- * mime.type
+ * mime.type. In addition, mime.extension is set if a common file extension is known.
* </p>
*
* <p>
- * The following MIME Types are supported:
+ * MIME Type detection is performed by Apache Tika; more information about
+ * detection is available at http://tika.apache.org.
*
* <ul>
- * <li>application/gzip</li>
- * <li>application/bzip2</li>
* <li>application/flowfile-v3</li>
- * <li>application/flowfile-v1 (requires Identify TAR be set to true)</li>
- * <li>application/xml</li>
- * <li>video/mp4</li>
- * <li>video/x-m4v</li>
- * <li>video/mp4a-latm</li>
- * <li>video/quicktime</li>
- * <li>video/mpeg</li>
- * <li>audio/wav</li>
- * <li>audio/mp3</li>
- * <li>image/bmp</li>
- * <li>image/png</li>
- * <li>image/jpg</li>
- * <li>image/gif</li>
- * <li>image/tif</li>
- * <li>application/vnd.ms-works</li>
- * <li>application/msexcel</li>
- * <li>application/mspowerpoint</li>
- * <li>application/msaccess</li>
- * <li>application/x-ms-wmv</li>
- * <li>application/pdf</li>
- * <li>application/x-rpm</li>
- * <li>application/tar</li>
- * <li>application/x-7z-compressed</li>
- * <li>application/java-archive</li>
- * <li>application/zip</li>
- * <li>application/x-lzh</li>
+ * <li>application/flowfile-v1</li>
* </ul>
* </p>
*/
@@ -97,128 +79,40 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@Tags({"compression", "gzip", "bzip2", "zip", "MIME", "mime.type", "file", "identify"})
@CapabilityDescription("Attempts to identify the MIME Type used for a FlowFile. If the MIME Type can be identified, "
+ "an attribute with the name 'mime.type' is added with the value being the MIME Type. If the MIME Type cannot be determined, "
- + "the value will be set to 'application/octet-stream'. Some MIME Types require reading a significant amount of data; for these MIME Types, their identification "
- + "is optional. The algorithm may have to read the entire contents of the file for each type of identification.")
+ + "the value will be set to 'application/octet-stream'. In addition, the attribute mime.extension will be set if a common file "
+ + "extension for the MIME Type is known.")
public class IdentifyMimeType extends AbstractProcessor {
- public static final PropertyDescriptor IDENTIFY_ZIP = new PropertyDescriptor.Builder()
- .name("Identify ZIP")
- .description("Determines whether or not to attempt in depth identification of ZIP MIME types")
- .required(true)
- .allowableValues("true", "false")
- .defaultValue("false")
- .build();
- public static final PropertyDescriptor IDENTIFY_TAR = new PropertyDescriptor.Builder()
- .name("Identify TAR")
- .description("Determines whether or not to attempt in depth identification of TAR MIME types")
- .required(true)
- .allowableValues("true", "false")
- .defaultValue("false")
- .build();
-
public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success").description("All FlowFiles are routed to success").build();
+ public static final MediaType FLOWFILE_V1 = new MediaType("application", "flowfile-v1");
+ public static final MediaType FLOWFILE_V3 = new MediaType("application", "flowfile-v3");
+
private Set<Relationship> relationships;
- private List<PropertyDescriptor> properties;
- private final List<MagicHeader> magicHeaders;
- private final List<MagicHeader> zipMagicHeaders;
- private final List<MagicHeader> tarMagicHeaders;
- private final List<ContentScanningMimeTypeIdentifier> contentScanners;
- private final int magicHeaderMaxLength;
+ private final TikaConfig config;
+ private final Detector detector;
public IdentifyMimeType() {
- // compile a list of Magic Header detectors
- final List<MagicHeader> headers = new ArrayList<>();
- headers.add(new SimpleMagicHeader("application/gzip", new byte[]{0x1f, (byte) 0x8b}));
- headers.add(new SimpleMagicHeader("application/bzip2", new byte[]{0x42, 0x5a}));
- headers.add(new SimpleMagicHeader("application/flowfile-v3", FlowFilePackagerV3.MAGIC_HEADER));
- headers.add(new SimpleMagicHeader("application/xml", new byte[]{0x3c, 0x3f, 0x78, 0x6d, 0x6c, 0x20}));
- headers.add(new SimpleMagicHeader("video/mp4", new byte[]{0, 0, 0, 0x14, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6F, 0x6D}));
- headers.add(new SimpleMagicHeader("video/mp4", new byte[]{0, 0, 0, 0x14, 0x66, 0x74, 0x79, 0x70, 0x33, 0x67, 0x70, 0x35}));
- headers.add(new SimpleMagicHeader("video/mp4", new byte[]{0, 0, 0, 0x14, 0x66, 0x74, 0x79, 0x70, 0X4d, 0X53, 0X4e, 0X56, 0X01, 0X29, 0, 0X46, 0X4d, 0X53, 0X4e, 0X56, 0X6d, 0X70, 0X34, 0X32}));
- headers.add(new SimpleMagicHeader("video/x-m4v", new byte[]{0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6D, 0x70, 0x34, 0x32}));
- headers.add(new SimpleMagicHeader("video/mp4a-latm", new byte[]{0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x4D, 0x34, 0x41, 0x20}));
- headers.add(new SimpleMagicHeader("video/quicktime", new byte[]{0, 0, 0, 0x14, 0x66, 0x74, 0x79, 0x70, 0x71, 0x74, 0x20, 0x20}));
- headers.add(new SimpleMagicHeader("video/quicktime", new byte[]{0x6D, 0x6F, 0x6F, 0x76}, 4));
- headers.add(new SimpleMagicHeader("audio/mp3", new byte[]{0x49, 0x44, 0x33}));
- headers.add(new SimpleMagicHeader("image/bmp", new byte[]{0x42, 0x4D}));
- headers.add(new SimpleMagicHeader("image/png", new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}));
- headers.add(new SimpleMagicHeader("image/jpg", new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF}));
- headers.add(new SimpleMagicHeader("image/gif", new byte[]{0x47, 0x49, 0x46, 0x38, 0x37, 0x61}));
- headers.add(new SimpleMagicHeader("image/gif", new byte[]{0x47, 0x49, 0x46, 0x38, 0x39, 0x61}));
- headers.add(new SimpleMagicHeader("image/tif", new byte[]{0x49, 0x20, 0x49}));
- headers.add(new SimpleMagicHeader("image/tif", new byte[]{0x49, 0x49, 0x2A, 0x00}));
- headers.add(new SimpleMagicHeader("image/tif", new byte[]{0x4D, 0x4D, 0x00, 0x2A}));
- headers.add(new SimpleMagicHeader("image/tif", new byte[]{0x4D, 0x4D, 0x00, 0x2B}));
- headers.add(new SimpleMagicHeader("application/vnd.ms-works", new byte[]{(byte) 0xFF, 0x00, 0x02, 0x00, 0x04, 0x04, 0x05, 0x54, 0x02, 0x00}));
- headers.add(new SimpleMagicHeader("application/msexcel", new byte[]{0x09, 0x08, 0x10, 0, 0, 0x06, 0x05, 0}, 512));
- headers.add(new SimpleMagicHeader("application/mspowerpoint", new byte[]{0x00, 0x6E, 0x1E, (byte) 0xF0}, 512));
- headers.add(new SimpleMagicHeader("application/mspowerpoint", new byte[]{0x0F, 0x00, (byte) 0xE8, 0x03}, 512));
- headers.add(new SimpleMagicHeader("application/mspowerpoint", new byte[]{(byte) 0xA0, 0x46, 0x1D, (byte) 0xF0}, 512));
- headers.add(new CompoundMagicHeader("application/mspowerpoint",
- new SimpleMagicHeader("", new byte[]{(byte) 0xFD, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}, 512),
- new SimpleMagicHeader("", new byte[]{0x00, 0x00, 0x00}, 517)));
- headers.add(new SimpleMagicHeader("application/msaccess", new byte[]{0x00, 0x01, 0x00, 0x00, 0x53, 0x74, 0x61, 0x6E, 0x64, 0x61, 0x72, 0x64, 0x20, 0x41, 0x43, 0x45, 0x20, 0x44, 0x42}));
- headers.add(new SimpleMagicHeader("application/msaccess", new byte[]{0x00, 0x01, 0x00, 0x00, 0x53, 0x74, 0x61, 0x6E, 0x64, 0x61, 0x72, 0x64, 0x20, 0x4A, 0x65, 0x74, 0x20, 0x44, 0x42}));
- for (byte b : new byte[]{0x10, 0x1F, 0x22, 0x23, 0x28, 0x29}) {
- headers.add(new SimpleMagicHeader("application/msaccess", new byte[]{(byte) 0xFD, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, b, 0x00}, 512));
- headers.add(new SimpleMagicHeader("application/msaccess", new byte[]{(byte) 0xFD, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, b, 0x02}, 512));
- }
- headers.add(new SimpleMagicHeader("application/x-ms-wmv", new byte[]{0x30, 0x26, (byte) 0xB2, 0x75, (byte) 0x8E, 0x66, (byte) 0xCF, 0x11, (byte) 0xA6, (byte) 0xD9, 0x00, (byte) 0xAA, 0x00, 0x62, (byte) 0xCE, 0x6C}));
- headers.add(new SimpleMagicHeader("application/pdf", new byte[]{0x25, 0x50, 0x44, 0x46}));
- headers.add(new SimpleMagicHeader("application/x-rpm", new byte[]{(byte) 0xED, (byte) 0xAB, (byte) 0xEE, (byte) 0xDB}));
- headers.add(new SimpleMagicHeader("application/x-7z-compressed", new byte[]{0x37, 0x7A, (byte) 0xBC, (byte) 0xAF, 0x27, 0x1C}));
- headers.add(new SimpleMagicHeader("application/java-archive", new byte[]{0x4A, 0x41, 0x52, 0x43, 0x53, 0x00}));
- headers.add(new SimpleMagicHeader("application/java-archive", new byte[]{0x50, 0x4B, 0x03, 0x04, 0x14, 0x00, 0x08}));
- headers.add(new SimpleMagicHeader("application/java-archive", new byte[]{0x50, 0x4B, 0x03, 0x04, (byte) 0xA0, 0x00, 0x00}));
- headers.add(new SimpleMagicHeader("application/x-lzh", new byte[]{0x2D, 0x6C, 0x68}, 2));
- headers.add(new CompoundMagicHeader("audio/wav",
- new SimpleMagicHeader("", new byte[]{0x52, 0x49, 0x46, 0x46}),
- new SimpleMagicHeader("", new byte[]{0x57, 0x41, 0x56, 0x45, 0x66, 0x6D, 0x74, 0x20}, 8)));
- for (int nibble = 0xB0; nibble <= 0xBF; nibble++) {
- headers.add(new SimpleMagicHeader("video/mpeg", new byte[]{0x00, 0x00, 0x01, (byte) nibble}));
- }
- this.magicHeaders = Collections.unmodifiableList(headers);
-
- // additional Magic Header detectors that will be turned off based on property settings
- final List<MagicHeader> zipHeaders = new ArrayList<>();
- zipHeaders.add(new SimpleMagicHeader("application/zip", new byte[]{0x50, 0x4B, 0x03, 0x04}));
- this.zipMagicHeaders = Collections.unmodifiableList(zipHeaders);
- final List<MagicHeader> tarHeaders = new ArrayList<>();
- tarHeaders.add(new SimpleMagicHeader("application/tar", new byte[]{0x75, 0x73, 0x74, 0x61, 0x72}, 257));
- this.tarMagicHeaders = Collections.unmodifiableList(tarHeaders);
-
- // determine the max length that we need to buffer for magic headers
- int max = 0;
- for (final MagicHeader header : magicHeaders) {
- max = Math.max(max, header.getRequiredBufferLength());
- }
- for (final MagicHeader header : zipMagicHeaders) {
- max = Math.max(max, header.getRequiredBufferLength());
- }
- for (final MagicHeader header : tarMagicHeaders) {
- max = Math.max(max, header.getRequiredBufferLength());
- }
- this.magicHeaderMaxLength = max;
-
- // create list of Content Scanners
- final List<ContentScanningMimeTypeIdentifier> scanningIdentifiers = new ArrayList<>();
- scanningIdentifiers.add(new ZipIdentifier());
- scanningIdentifiers.add(new TarIdentifier());
- this.contentScanners = Collections.unmodifiableList(scanningIdentifiers);
+ // Setup Tika
+ this.config = TikaConfig.getDefaultConfig();
+ DefaultDetector ddetector = new DefaultDetector();
+
+ // Create list of detectors, preferring our custom detectors first
+ List<Detector> detectors = new ArrayList<>();
+ detectors.add(getFlowFileV3Detector());
+ detectors.add(getFlowFileV1Detector());
+ detectors.addAll(ddetector.getDetectors());
+
+ CompositeDetector compositeDetector = new CompositeDetector(detectors);
+ this.detector = compositeDetector;
}
@Override
protected void init(final ProcessorInitializationContext context) {
- final Set<Relationship> relationships = new HashSet<>();
- relationships.add(REL_SUCCESS);
- this.relationships = Collections.unmodifiableSet(relationships);
-
- final List<PropertyDescriptor> properties = new ArrayList<>();
- properties.add(IDENTIFY_ZIP);
- properties.add(IDENTIFY_TAR);
- this.properties = Collections.unmodifiableList(properties);
+ final Set<Relationship> rels = new HashSet<>();
+ rels.add(REL_SUCCESS);
+ this.relationships = Collections.unmodifiableSet(rels);
}
@Override
@@ -227,11 +121,6 @@ public class IdentifyMimeType extends AbstractProcessor {
}
@Override
- protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- return properties;
- }
-
- @Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
FlowFile flowFile = session.get();
if (flowFile == null) {
@@ -239,87 +128,39 @@ public class IdentifyMimeType extends AbstractProcessor {
}
final ProcessorLog logger = getLogger();
- final boolean identifyZip = context.getProperty(IDENTIFY_ZIP).asBoolean();
- final boolean identifyTar = context.getProperty(IDENTIFY_TAR).asBoolean();
final ObjectHolder<String> mimeTypeRef = new ObjectHolder<>(null);
+ final ObjectHolder<String> extensionRef = new ObjectHolder<>(null);
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream stream) throws IOException {
try (final InputStream in = new BufferedInputStream(stream)) {
- // read in up to magicHeaderMaxLength bytes
- in.mark(magicHeaderMaxLength);
- byte[] header = new byte[magicHeaderMaxLength];
- for (int i = 0; i < header.length; i++) {
- final int next = in.read();
- if (next >= 0) {
- header[i] = (byte) next;
- } else if (i == 0) {
- header = new byte[0];
- } else {
- final byte[] newBuffer = new byte[i - 1];
- System.arraycopy(header, 0, newBuffer, 0, i - 1);
- header = newBuffer;
- break;
- }
- }
- in.reset();
-
- for (final MagicHeader magicHeader : magicHeaders) {
- if (magicHeader.matches(header)) {
- mimeTypeRef.set(magicHeader.getMimeType());
- return;
- }
- }
-
- if (!identifyZip) {
- for (final MagicHeader magicHeader : zipMagicHeaders) {
- if (magicHeader.matches(header)) {
- mimeTypeRef.set(magicHeader.getMimeType());
- return;
- }
- }
- }
-
- if (!identifyTar) {
- for (final MagicHeader magicHeader : tarMagicHeaders) {
- if (magicHeader.matches(header)) {
- mimeTypeRef.set(magicHeader.getMimeType());
- return;
- }
- }
+ TikaInputStream tikaStream = TikaInputStream.get(in);
+ Metadata metadata = new Metadata();
+ // Get mime type
+ MediaType mediatype = detector.detect(tikaStream, metadata);
+ mimeTypeRef.set(mediatype.toString());
+ // Get common file extension
+ try {
+ MimeType mimetype;
+ mimetype = config.getMimeRepository().forName(mediatype.toString());
+ extensionRef.set(mimetype.getExtension());
+ } catch (MimeTypeException ex) {
+ logger.warn("MIME type detection failed: {}", new Object[]{ex.toString()});
}
}
}
});
String mimeType = mimeTypeRef.get();
- if (mimeType == null) {
- for (final ContentScanningMimeTypeIdentifier scanningIdentifier : this.contentScanners) {
- if (scanningIdentifier.isEnabled(context)) {
- session.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(final InputStream in) throws IOException {
- String mimeType = scanningIdentifier.getMimeType(in);
- if (mimeType != null) {
- mimeTypeRef.set(mimeType);
- }
- }
- });
-
- if (mimeTypeRef.get() != null) {
- break;
- }
- }
- }
- }
-
- mimeType = mimeTypeRef.get();
+ String extension = extensionRef.get();
if (mimeType == null) {
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/octet-stream");
+ flowFile = session.putAttribute(flowFile, "mime.extension", "");
logger.info("Unable to identify MIME Type for {}; setting to application/octet-stream", new Object[]{flowFile});
} else {
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), mimeType);
+ flowFile = session.putAttribute(flowFile, "mime.extension", extension);
logger.info("Identified {} as having MIME Type {}", new Object[]{flowFile, mimeType});
}
@@ -327,148 +168,41 @@ public class IdentifyMimeType extends AbstractProcessor {
session.transfer(flowFile, REL_SUCCESS);
}
- private static interface ContentScanningMimeTypeIdentifier {
-
- boolean isEnabled(ProcessContext context);
-
- String getMimeType(InputStream in) throws IOException;
- }
-
- private static class ZipIdentifier implements ContentScanningMimeTypeIdentifier {
-
- @Override
- public String getMimeType(final InputStream in) throws IOException {
- final ZipInputStream zipIn = new ZipInputStream(in);
- try {
- if (zipIn.getNextEntry() != null) {
- return "application/zip";
- }
- } catch (final Exception e) {
- }
- return null;
- }
-
- @Override
- public boolean isEnabled(final ProcessContext context) {
- return context.getProperty(IDENTIFY_ZIP).asBoolean();
- }
- }
-
- private static class TarIdentifier implements ContentScanningMimeTypeIdentifier {
-
- @Override
- public String getMimeType(final InputStream in) throws IOException {
- try (final TarArchiveInputStream tarIn = new TarArchiveInputStream(in)) {
- final TarArchiveEntry firstEntry = tarIn.getNextTarEntry();
- if (firstEntry != null) {
- if (firstEntry.getName().equals(FlowFilePackagerV1.FILENAME_ATTRIBUTES)) {
- final TarArchiveEntry secondEntry = tarIn.getNextTarEntry();
- if (secondEntry != null && secondEntry.getName().equals(FlowFilePackagerV1.FILENAME_CONTENT)) {
- return "application/flowfile-v1";
- }
- }
- return "application/tar";
- }
- } catch (final Exception e) {
- }
- return null;
- }
-
- @Override
- public boolean isEnabled(final ProcessContext context) {
- return context.getProperty(IDENTIFY_TAR).asBoolean();
- }
+ private Detector getFlowFileV3Detector() {
+ return new MagicDetector(FLOWFILE_V3, FlowFilePackagerV3.MAGIC_HEADER);
}
- private static interface MagicHeader {
-
- int getRequiredBufferLength();
-
- String getMimeType();
-
- boolean matches(final byte[] header);
+ private Detector getFlowFileV1Detector() {
+ return new FlowFileV1Detector();
}
- private static class SimpleMagicHeader implements MagicHeader {
-
- private final String mimeType;
- private final int offset;
- private final byte[] byteSequence;
-
- public SimpleMagicHeader(final String mimeType, final byte[] byteSequence) {
- this(mimeType, byteSequence, 0);
- }
-
- public SimpleMagicHeader(final String mimeType, final byte[] byteSequence, final int offset) {
- this.mimeType = mimeType;
- this.byteSequence = byteSequence;
- this.offset = offset;
- }
-
- @Override
- public int getRequiredBufferLength() {
- return byteSequence.length + offset;
- }
-
- @Override
- public String getMimeType() {
- return mimeType;
- }
+ private class FlowFileV1Detector implements Detector {
@Override
- public boolean matches(final byte[] header) {
- if (header.length < getRequiredBufferLength()) {
- return false;
+ public MediaType detect(InputStream in, Metadata mtdt) throws IOException {
+ // Sanity check the stream. This may not be a tarfile at all
+ in.mark(FlowFilePackagerV1.FILENAME_ATTRIBUTES.length());
+ byte[] bytes = new byte[FlowFilePackagerV1.FILENAME_ATTRIBUTES.length()];
+ in.read(bytes);
+ in.reset();
+
+ // Quick exit if the first filename is not correct
+ if (!Arrays.equals(bytes, FlowFilePackagerV1.FILENAME_ATTRIBUTES.getBytes())) {
+ return MediaType.OCTET_STREAM;
}
- for (int i = 0; i < byteSequence.length; i++) {
- if (byteSequence[i] != header[offset + i]) {
- return false;
- }
- }
-
- return true;
- }
- }
-
- private static class CompoundMagicHeader implements MagicHeader {
-
- private final MagicHeader[] headers;
- private final int requiredLength;
- private final String mimeType;
-
- public CompoundMagicHeader(final String mimeType, final MagicHeader... headers) {
- this.mimeType = mimeType;
- this.headers = headers;
-
- int max = 0;
- for (final MagicHeader header : headers) {
- max = Math.max(max, header.getRequiredBufferLength());
- }
-
- this.requiredLength = max;
- }
-
- @Override
- public int getRequiredBufferLength() {
- return requiredLength;
- }
-
- @Override
- public String getMimeType() {
- return mimeType;
- }
-
- @Override
- public boolean matches(final byte[] header) {
- for (final MagicHeader mh : headers) {
- if (!mh.matches(header)) {
- return false;
+ // More in-depth detection
+ final TarArchiveInputStream tarIn = new TarArchiveInputStream(in);
+ final TarArchiveEntry firstEntry = tarIn.getNextTarEntry();
+ if (firstEntry != null) {
+ if (firstEntry.getName().equals(FlowFilePackagerV1.FILENAME_ATTRIBUTES)) {
+ final TarArchiveEntry secondEntry = tarIn.getNextTarEntry();
+ if (secondEntry != null && secondEntry.getName().equals(FlowFilePackagerV1.FILENAME_CONTENT)) {
+ return FLOWFILE_V1;
+ }
}
}
-
- return true;
+ return MediaType.OCTET_STREAM;
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/16fb2b82/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
index c72bff7..40b03b4 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
@@ -57,63 +57,16 @@ public class TestIdentifyMimeType {
final Map<String, String> expectedMimeTypes = new HashMap<>();
expectedMimeTypes.put("1.7z", "application/x-7z-compressed");
- expectedMimeTypes.put("1.mdb", "application/msaccess");
- expectedMimeTypes.put("1.txt.bz2", "application/bzip2");
+ expectedMimeTypes.put("1.mdb", "application/x-msaccess");
+ expectedMimeTypes.put("1.txt", "text/plain");
+ expectedMimeTypes.put("1.txt.bz2", "application/x-bzip2");
expectedMimeTypes.put("1.txt.gz", "application/gzip");
expectedMimeTypes.put("1.zip", "application/zip");
expectedMimeTypes.put("bgBannerFoot.png", "image/png");
- expectedMimeTypes.put("blueBtnBg.jpg", "image/jpg");
+ expectedMimeTypes.put("blueBtnBg.jpg", "image/jpeg");
expectedMimeTypes.put("1.pdf", "application/pdf");
expectedMimeTypes.put("grid.gif", "image/gif");
- expectedMimeTypes.put("1.tar", "application/octet-stream"); //wrong ID without IDENTIFY_TAR
- expectedMimeTypes.put("1.jar", "application/java-archive");
- expectedMimeTypes.put("1.xml", "application/xml");
- expectedMimeTypes.put("flowfilev3", "application/flowfile-v3");
- expectedMimeTypes.put("flowfilev1.tar", "application/tar"); //wrong ID without IDENTIFY_TAR
-
- final List<MockFlowFile> filesOut = runner.getFlowFilesForRelationship(IdentifyMimeType.REL_SUCCESS);
- for (final MockFlowFile file : filesOut) {
- final String filename = file.getAttribute(CoreAttributes.FILENAME.key());
- final String mimeType = file.getAttribute(CoreAttributes.MIME_TYPE.key());
- final String expected = expectedMimeTypes.get(filename);
- assertEquals("Expected " + file + " to have MIME Type " + expected + ", but it was " + mimeType, expected, mimeType);
- }
- }
-
- @Test
- public void testFilesWithIdentifyTarAndZip() throws IOException {
- final TestRunner runner = TestRunners.newTestRunner(new IdentifyMimeType());
- runner.setProperty(IdentifyMimeType.IDENTIFY_TAR.getName(), "true");
- runner.setProperty(IdentifyMimeType.IDENTIFY_ZIP.getName(), "true");
-
- final File dir = new File("src/test/resources/TestIdentifyMimeType");
- final File[] files = dir.listFiles();
- int fileCount = 0;
- for (final File file : files) {
- if (file.isDirectory()) {
- continue;
- }
-
- runner.enqueue(file.toPath());
- fileCount++;
- }
-
- runner.setThreadCount(4);
- runner.run(fileCount);
-
- runner.assertAllFlowFilesTransferred(IdentifyMimeType.REL_SUCCESS, fileCount);
-
- final Map<String, String> expectedMimeTypes = new HashMap<>();
- expectedMimeTypes.put("1.7z", "application/x-7z-compressed");
- expectedMimeTypes.put("1.mdb", "application/msaccess");
- expectedMimeTypes.put("1.txt.bz2", "application/bzip2");
- expectedMimeTypes.put("1.txt.gz", "application/gzip");
- expectedMimeTypes.put("1.zip", "application/zip");
- expectedMimeTypes.put("bgBannerFoot.png", "image/png");
- expectedMimeTypes.put("blueBtnBg.jpg", "image/jpg");
- expectedMimeTypes.put("1.pdf", "application/pdf");
- expectedMimeTypes.put("grid.gif", "image/gif");
- expectedMimeTypes.put("1.tar", "application/tar");
+ expectedMimeTypes.put("1.tar", "application/x-tar");
expectedMimeTypes.put("1.jar", "application/java-archive");
expectedMimeTypes.put("1.xml", "application/xml");
expectedMimeTypes.put("flowfilev3", "application/flowfile-v3");
@@ -127,5 +80,4 @@ public class TestIdentifyMimeType {
assertEquals("Expected " + file + " to have MIME Type " + expected + ", but it was " + mimeType, expected, mimeType);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/16fb2b82/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar
index 6be4fcb..f60118c 100755
Binary files a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar and b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.tar differ
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/16fb2b82/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.txt
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.txt b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.txt
new file mode 100755
index 0000000..10be193
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestIdentifyMimeType/1.txt
@@ -0,0 +1 @@
+Test IdentifyMimeType
\ No newline at end of file
[18/50] incubator-nifi git commit: NIFI-296: Minor updates after code
review
Posted by mc...@apache.org.
NIFI-296: Minor updates after code review
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/b418b890
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/b418b890
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/b418b890
Branch: refs/heads/NIFI-353
Commit: b418b890a63e328136b9f1a7e15e15b01fd8883a
Parents: 16fb2b8
Author: Adam Lamar <ad...@gmail.com>
Authored: Fri Feb 20 09:28:00 2015 +0000
Committer: Adam Lamar <ad...@gmail.com>
Committed: Fri Feb 20 09:28:00 2015 +0000
----------------------------------------------------------------------
.../org/apache/nifi/processors/standard/IdentifyMimeType.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/b418b890/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
index c9cb1cd..fd3d4ae 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
@@ -42,6 +42,7 @@ import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.util.FlowFilePackagerV1;
import org.apache.nifi.util.FlowFilePackagerV3;
import org.apache.nifi.util.ObjectHolder;
@@ -146,7 +147,7 @@ public class IdentifyMimeType extends AbstractProcessor {
mimetype = config.getMimeRepository().forName(mediatype.toString());
extensionRef.set(mimetype.getExtension());
} catch (MimeTypeException ex) {
- logger.warn("MIME type detection failed: {}", new Object[]{ex.toString()});
+ logger.warn("MIME type detection failed: {}", new Object[]{ex});
}
}
}
@@ -183,7 +184,7 @@ public class IdentifyMimeType extends AbstractProcessor {
// Sanity check the stream. This may not be a tarfile at all
in.mark(FlowFilePackagerV1.FILENAME_ATTRIBUTES.length());
byte[] bytes = new byte[FlowFilePackagerV1.FILENAME_ATTRIBUTES.length()];
- in.read(bytes);
+ StreamUtils.fillBuffer(in, bytes, false);
in.reset();
// Quick exit if the first filename is not correct
[40/50] incubator-nifi git commit: Adjusting handling of map to cache
data items on an instance basis.
Posted by mc...@apache.org.
Adjusting handling of map to cache data items on an instance basis.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/973b4933
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/973b4933
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/973b4933
Branch: refs/heads/NIFI-353
Commit: 973b493386c71017f9baa233d4ec178251e64f53
Parents: 484687a
Author: Aldrin Piri <al...@gmail.com>
Authored: Sun Mar 1 16:31:32 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Sun Mar 1 16:31:32 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 14 ++---
.../nifi/processors/standard/SplitJson.java | 55 ++++++++++----------
2 files changed, 34 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/973b4933/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 81c9bbe..64f6e0d 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -40,7 +40,6 @@ import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.util.ObjectHolder;
-import org.apache.nifi.util.Tuple;
import java.io.IOException;
import java.io.OutputStream;
@@ -96,7 +95,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
private Set<Relationship> relationships;
private List<PropertyDescriptor> properties;
- private ConcurrentMap<String, Tuple<String, JsonPath>> cachedJsonPathMap = new ConcurrentHashMap<>();
+ private final ConcurrentMap<String, JsonPath> cachedJsonPathMap = new ConcurrentHashMap<>();
@Override
protected void init(final ProcessorInitializationContext context) {
@@ -153,13 +152,12 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
.addValidator(new JsonPathValidator() {
@Override
public void cacheComputedValue(String subject, String input, JsonPath computedJsonPath) {
- cachedJsonPathMap.put(subject, new Tuple<>(input, computedJsonPath));
-
+ cachedJsonPathMap.put(input, computedJsonPath);
}
@Override
public boolean isStale(String subject, String input) {
- return cachedJsonPathMap.get(subject) == null;
+ return cachedJsonPathMap.get(input) == null;
}
})
.required(false)
@@ -171,7 +169,9 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
if (descriptor.isDynamic()) {
if (!StringUtils.equals(oldValue, newValue)) {
- cachedJsonPathMap.remove(descriptor.getName());
+ if (oldValue != null) {
+ cachedJsonPathMap.remove(oldValue);
+ }
}
}
}
@@ -185,7 +185,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
public void onRemoved(ProcessContext processContext) {
for (PropertyDescriptor propertyDescriptor : getPropertyDescriptors()) {
if (propertyDescriptor.isDynamic()) {
- cachedJsonPathMap.remove(propertyDescriptor.getName());
+ cachedJsonPathMap.remove(processContext.getProperty(propertyDescriptor).getValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/973b4933/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
index 7bb8c4e..5a193a1 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -26,8 +26,9 @@ import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnRemoved;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.ProcessContext;
@@ -35,13 +36,13 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
@EventDriven
@SideEffectFree
@@ -56,18 +57,8 @@ public class SplitJson extends AbstractJsonPathProcessor {
public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder()
.name("JsonPath Expression")
.description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.required(true)
- .addValidator(new JsonPathValidator() {
- @Override
- public void cacheComputedValue(String subject, String input, JsonPath computedJson) {
- JSON_PATH_MAP.put(input, computedJson);
- }
-
- @Override
- public boolean isStale(String subject, String input) {
- return JSON_PATH_MAP.get(input) == null;
- }
- })
.build();
public static final Relationship REL_ORIGINAL = new Relationship.Builder().name("original").description("The original FlowFile that was split into segments. If the FlowFile fails processing, nothing will be sent to this relationship").build();
@@ -77,7 +68,7 @@ public class SplitJson extends AbstractJsonPathProcessor {
private List<PropertyDescriptor> properties;
private Set<Relationship> relationships;
- private static final ConcurrentMap<String, JsonPath> JSON_PATH_MAP = new ConcurrentHashMap();
+ private final AtomicReference<JsonPath> JSON_PATH_REF = new AtomicReference<>();
@Override
protected void init(final ProcessorInitializationContext context) {
@@ -106,21 +97,30 @@ public class SplitJson extends AbstractJsonPathProcessor {
public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
if (descriptor.equals(ARRAY_JSON_PATH_EXPRESSION)) {
if (!StringUtils.equals(oldValue, newValue)) {
- // clear the cached item
- JSON_PATH_MAP.remove(oldValue);
+ if (oldValue != null) {
+ // clear the cached item
+ JSON_PATH_REF.set(null);
+ }
}
}
}
- /**
- * Provides cleanup of the map for any JsonPath values that may have been created. This will remove common values
- * shared between multiple instances, but will be regenerated when the next validation cycle occurs as a result of
- * isStale()
- */
- @OnRemoved
- public void onRemoved(ProcessContext processContext) {
- String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
- JSON_PATH_MAP.remove(jsonPathExpression);
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ JsonPathValidator validator = new JsonPathValidator() {
+ @Override
+ public void cacheComputedValue(String subject, String input, JsonPath computedJson) {
+ JSON_PATH_REF.set(computedJson);
+ }
+
+ @Override
+ public boolean isStale(String subject, String input) {
+ return JSON_PATH_REF.get() == null;
+ }
+ };
+
+ String value = validationContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
+ return Collections.singleton(validator.validate(ARRAY_JSON_PATH_EXPRESSION.getName(), value, validationContext));
}
@Override
@@ -141,8 +141,7 @@ public class SplitJson extends AbstractJsonPathProcessor {
return;
}
- String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
- final JsonPath jsonPath = JSON_PATH_MAP.get(jsonPathExpression);
+ final JsonPath jsonPath = JSON_PATH_REF.get();
final List<FlowFile> segments = new ArrayList<>();
[07/50] incubator-nifi git commit: Providing a SplitJson processor
which will break JSON Arrays into their individual elements. Refactored
supporting JsonUtils code and EvaluateJsonPath to reuse common functionality.
Posted by mc...@apache.org.
Providing a SplitJson processor which will break JSON Arrays into their individual elements. Refactored supporting JsonUtils code and EvaluateJsonPath to reuse common functionality.
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/2e05dcbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/2e05dcbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/2e05dcbb
Branch: refs/heads/NIFI-353
Commit: 2e05dcbbfdb37843456dff121d9878d8679d7067
Parents: 59ad194
Author: Aldrin Piri <al...@gmail.com>
Authored: Tue Feb 17 15:15:53 2015 -0500
Committer: Aldrin Piri <al...@gmail.com>
Committed: Tue Feb 17 15:17:12 2015 -0500
----------------------------------------------------------------------
.../processors/standard/EvaluateJsonPath.java | 8 +-
.../nifi/processors/standard/SplitJson.java | 140 +++++++++++++++++++
.../processors/standard/util/JsonUtils.java | 8 ++
.../org.apache.nifi.processor.Processor | 1 +
.../standard/TestEvaluateJsonPath.java | 3 +-
.../nifi/processors/standard/TestSplitJson.java | 115 +++++++++++++++
6 files changed, 268 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2e05dcbb/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
index 0bdd20c..8cdf1a2 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateJsonPath.java
@@ -192,7 +192,7 @@ public class EvaluateJsonPath extends AbstractProcessor {
final ObjectHolder<Object> resultHolder = new ObjectHolder<>(null);
try {
Object result = documentContext.read(jsonPathExp);
- if (returnType.equals(RETURN_TYPE_SCALAR) && !isScalar(result)) {
+ if (returnType.equals(RETURN_TYPE_SCALAR) && !JsonUtils.isJsonScalar(result)) {
logger.error("Unable to return a scalar value for the expression {} for FlowFile {}. Evaluated value was {}. Transferring to {}.",
new Object[]{jsonPathExp.getPath(), flowFile.getId(), result.toString(), REL_FAILURE.getName()});
processSession.transfer(flowFile, REL_FAILURE);
@@ -233,15 +233,11 @@ public class EvaluateJsonPath extends AbstractProcessor {
}
private static String getResultRepresentation(Object jsonPathResult) {
- if (isScalar(jsonPathResult)) {
+ if (JsonUtils.isJsonScalar(jsonPathResult)) {
return jsonPathResult.toString();
}
return JsonUtils.JSON_PROVIDER.toJson(jsonPathResult);
}
- private static boolean isScalar(Object obj) {
- return (obj instanceof String);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2e05dcbb/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
new file mode 100644
index 0000000..e589b48
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.logging.ProcessorLog;
+import org.apache.nifi.processor.*;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processors.standard.util.JsonUtils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@Tags({"json", "split", "jsonpath"})
+@CapabilityDescription("Splits a JSON File into multiple, separate FlowFiles for an array element specified by a JsonPath expression. "
+ + "Each generated FlowFile is comprised of an element of the specified array and transferred to relationship 'split, "
+ + "with the original file transferred to the 'original' relationship. If the specified JsonPath is not found or "
+ + "does not evaluate to an array element, the original file is routed to 'failure' and no files are generated.")
+public class SplitJson extends AbstractProcessor {
+
+ public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder()
+ .name("JsonPath Expression")
+ .description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
+ .required(true)
+ .addValidator(JsonUtils.JSON_PATH_VALIDATOR)
+ .build();
+
+ public static final Relationship REL_ORIGINAL = new Relationship.Builder().name("original").description("The original FlowFile that was split into segments. If the FlowFile fails processing, nothing will be sent to this relationship").build();
+ public static final Relationship REL_SPLIT = new Relationship.Builder().name("split").description("All segments of the original FlowFile will be routed to this relationship").build();
+ public static final Relationship REL_FAILURE = new Relationship.Builder().name("failure").description("If a FlowFile fails processing for any reason (for example, the FlowFile is not valid JSON or the specified path does not exist), it will be routed to this relationship").build();
+
+ private List<PropertyDescriptor> properties;
+ private Set<Relationship> relationships;
+
+ @Override
+ protected void init(final ProcessorInitializationContext context) {
+ final List<PropertyDescriptor> properties = new ArrayList<>();
+ properties.add(ARRAY_JSON_PATH_EXPRESSION);
+ this.properties = Collections.unmodifiableList(properties);
+
+ final Set<Relationship> relationships = new HashSet<>();
+ relationships.add(REL_ORIGINAL);
+ relationships.add(REL_SPLIT);
+ relationships.add(REL_FAILURE);
+ this.relationships = Collections.unmodifiableSet(relationships);
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext processContext, final ProcessSession processSession) {
+ final FlowFile original = processSession.get();
+ if (original == null) {
+ return;
+ }
+
+ final ProcessorLog logger = getLogger();
+
+
+ final DocumentContext documentContext = JsonUtils.validateAndEstablishJsonContext(processSession, original);
+
+ if (documentContext == null) {
+ logger.error("FlowFile {} did not have valid JSON content.", new Object[]{original});
+ processSession.transfer(original, REL_FAILURE);
+ return;
+ }
+
+ final String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
+ final JsonPath jsonPath = JsonPath.compile(jsonPathExpression);
+
+ final List<FlowFile> segments = new ArrayList<>();
+
+ Object jsonPathResult = documentContext.read(jsonPath);
+
+ if (!(jsonPathResult instanceof List)) {
+ logger.error("The evaluated value {} of {} was not an array compatible type and cannot be split.",
+ new Object[]{jsonPathResult, jsonPath.getPath()});
+ processSession.transfer(original, REL_FAILURE);
+ return;
+ }
+
+ List resultList = (List) jsonPathResult;
+
+ for (final Object resultSegment : resultList) {
+ FlowFile split = processSession.create(original);
+ split = processSession.write(split, new OutputStreamCallback() {
+ @Override
+ public void process(OutputStream out) throws IOException {
+ String resultSegmentContent;
+ if (JsonUtils.isJsonScalar(resultSegment)) {
+ resultSegmentContent = resultSegment.toString();
+ } else {
+ resultSegmentContent = JsonUtils.JSON_PROVIDER.toJson(resultSegment);
+ }
+ out.write(resultSegmentContent.getBytes(StandardCharsets.UTF_8));
+ }
+ });
+ segments.add(split);
+ }
+
+ processSession.transfer(segments, REL_SPLIT);
+ processSession.transfer(original, REL_ORIGINAL);
+ logger.info("Split {} into {} FlowFiles", new Object[]{original, segments.size()});
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2e05dcbb/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
index 6eb567e..0bf33dd 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JsonUtils.java
@@ -35,6 +35,8 @@ import org.apache.nifi.util.ObjectHolder;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.util.List;
+import java.util.Map;
/**
* Provides utilities for interacting with JSON elements and JsonPath expressions and results
@@ -101,4 +103,10 @@ public class JsonUtils {
return isValid;
}
+
+ public static boolean isJsonScalar(Object obj) {
+ // For the default provider, a Map or List is able to be handled as a JSON entity
+ return !(obj instanceof Map || obj instanceof List);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2e05dcbb/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index 8a1fd74..17b5364 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -55,6 +55,7 @@ org.apache.nifi.processors.standard.ScanAttribute
org.apache.nifi.processors.standard.ScanContent
org.apache.nifi.processors.standard.SegmentContent
org.apache.nifi.processors.standard.SplitContent
+org.apache.nifi.processors.standard.SplitJson
org.apache.nifi.processors.standard.SplitText
org.apache.nifi.processors.standard.SplitXml
org.apache.nifi.processors.standard.TransformXml
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2e05dcbb/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
index c873969..b7b5103 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateJsonPath.java
@@ -18,6 +18,7 @@ package org.apache.nifi.processors.standard;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.StringUtils;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Assert;
@@ -158,7 +159,7 @@ public class TestEvaluateJsonPath {
testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(expectedRel).get(0);
Assert.assertEquals("Transferred flow file did not have the correct result for id attribute", "54df94072d5dbf7dc6340cc5", out.getAttribute(jsonPathIdAttrKey));
- Assert.assertEquals("Transferred flow file did not have the correct result for name attribute", "", out.getAttribute(jsonPathNameAttrKey));
+ Assert.assertEquals("Transferred flow file did not have the correct result for name attribute", StringUtils.EMPTY, out.getAttribute(jsonPathNameAttrKey));
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/2e05dcbb/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
new file mode 100644
index 0000000..dd6fc6d
--- /dev/null
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestSplitJson.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class TestSplitJson {
+
+ private static final Path JSON_SNIPPET = Paths.get("src/test/resources/TestJson/json-sample.json");
+ private static final Path XML_SNIPPET = Paths.get("src/test/resources/TestXml/xml-snippet.xml");
+
+ @Test(expected = AssertionError.class)
+ public void testInvalidJsonPath() {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$..");
+
+ Assert.fail("An improper JsonPath expression was not detected as being invalid.");
+ }
+
+ @Test
+ public void testInvalidJsonDocument() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$");
+
+ testRunner.enqueue(XML_SNIPPET);
+ testRunner.run();
+
+ testRunner.assertAllFlowFilesTransferred(SplitJson.REL_FAILURE, 1);
+ final MockFlowFile out = testRunner.getFlowFilesForRelationship(SplitJson.REL_FAILURE).get(0);
+ // Verify that the content was unchanged
+ out.assertContentEquals(XML_SNIPPET);
+ }
+
+ @Test
+ public void testSplit_nonArrayResult() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$[0]._id");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ Relationship expectedRel = SplitJson.REL_FAILURE;
+
+ testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
+ final MockFlowFile out = testRunner.getFlowFilesForRelationship(expectedRel).get(0);
+ out.assertContentEquals(JSON_SNIPPET);
+ }
+
+ @Test
+ public void testSplit_arrayResult_oneValue() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$[0].range[?(@ == 0)]");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ testRunner.assertTransferCount(SplitJson.REL_ORIGINAL, 1);
+ testRunner.assertTransferCount(SplitJson.REL_SPLIT, 1);
+ testRunner.getFlowFilesForRelationship(SplitJson.REL_ORIGINAL).get(0).assertContentEquals(JSON_SNIPPET);
+ testRunner.getFlowFilesForRelationship(SplitJson.REL_SPLIT).get(0).assertContentEquals("0");
+ }
+
+ @Test
+ public void testSplit_arrayResult_multipleValues() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$[0].range");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ int numSplitsExpected = 10;
+
+ testRunner.assertTransferCount(SplitJson.REL_ORIGINAL, 1);
+ testRunner.assertTransferCount(SplitJson.REL_SPLIT, numSplitsExpected);
+ final MockFlowFile originalOut = testRunner.getFlowFilesForRelationship(SplitJson.REL_ORIGINAL).get(0);
+ originalOut.assertContentEquals(JSON_SNIPPET);
+ }
+
+ @Test
+ public void testSplit_arrayResult_nonScalarValues() throws Exception {
+ final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson());
+ testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$[*].name");
+
+ testRunner.enqueue(JSON_SNIPPET);
+ testRunner.run();
+
+ testRunner.assertTransferCount(SplitJson.REL_ORIGINAL, 1);
+ testRunner.assertTransferCount(SplitJson.REL_SPLIT, 7);
+ testRunner.getFlowFilesForRelationship(SplitJson.REL_ORIGINAL).get(0).assertContentEquals(JSON_SNIPPET);
+ testRunner.getFlowFilesForRelationship(SplitJson.REL_SPLIT).get(0).assertContentEquals("{\"first\":\"Shaffer\",\"last\":\"Pearson\"}");
+ }
+
+}